ref #2937 start GermanSL import
authorAndreas Müller <a.mueller@bgbm.org>
Thu, 8 Dec 2016 15:17:05 +0000 (16:17 +0100)
committerAndreas Müller <a.mueller@bgbm.org>
Thu, 8 Dec 2016 15:17:05 +0000 (16:17 +0100)
app-import/src/main/java/eu/etaxonomy/cdm/app/common/CdmDestinations.java
app-import/src/main/java/eu/etaxonomy/cdm/app/redlist/GermanSLActivator.java [new file with mode: 0644]
app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLImportConfigurator.java [new file with mode: 0644]
app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTaxonImport.java [new file with mode: 0644]
app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTaxonRelationImport.java [new file with mode: 0644]
app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTransformer.java [new file with mode: 0644]

index 3dada92a5db74a818ccc437fae41abe452aa8c76..bc15b8253fc41b53a8f7d5dc860ff3c609c7bfea 100644 (file)
@@ -243,6 +243,23 @@ public class CdmDestinations {
                return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null);
        }
 
+       public static ICdmDataSource cdm_redlist_germanSL_preview(){
+        DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
+        String cdmServer = "160.45.63.175";
+        String cdmDB = "cdm_rl_german_sl";
+        String cdmUserName = "edit";
+        return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null);
+    }
+
+    public static ICdmDataSource cdm_germanSL_production(){
+        DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
+        String cdmServer = "160.45.63.171";
+        String cdmDB = "cdm_production_rl_german_sl";
+        String cdmUserName = "edit";
+        return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null);
+    }
+
+
        public static ICdmDataSource cdm_cyprus_dev(){
                DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
                String cdmServer = "160.45.63.175";
diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/app/redlist/GermanSLActivator.java b/app-import/src/main/java/eu/etaxonomy/cdm/app/redlist/GermanSLActivator.java
new file mode 100644 (file)
index 0000000..cb07182
--- /dev/null
@@ -0,0 +1,127 @@
+/**
+* Copyright (C) 2016 EDIT
+* European Distributed Institute of Taxonomy
+* http://www.e-taxonomy.eu
+*
+* The contents of this file are subject to the Mozilla Public License Version 1.1
+* See LICENSE.TXT at the top of this package for the full license terms.
+*/
+package eu.etaxonomy.cdm.app.redlist;
+
+import java.net.URI;
+import java.util.UUID;
+
+import org.apache.log4j.Logger;
+
+import eu.etaxonomy.cdm.app.common.CdmDestinations;
+import eu.etaxonomy.cdm.database.DbSchemaValidation;
+import eu.etaxonomy.cdm.database.ICdmDataSource;
+import eu.etaxonomy.cdm.io.common.CdmDefaultImport;
+import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
+import eu.etaxonomy.cdm.io.redlist.germanSL.GermanSLImportConfigurator;
+import eu.etaxonomy.cdm.model.agent.Person;
+import eu.etaxonomy.cdm.model.reference.Reference;
+import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
+
+/**
+ * @author a.mueller
+ * @date 25.11.2016
+ *
+ */
+public class GermanSLActivator {
+    private static final Logger logger = Logger.getLogger(GermanSLActivator.class);
+
+    //database validation status (create, update, validate ...)
+    static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
+
+//    static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
+//    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql_test();
+    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_redlist_germanSL_preview();
+//    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_germanSL_production();
+
+    static String versionString = "version_1_3_4";
+
+    //feature tree uuid
+    public static final UUID featureTreeUuid = UUID.fromString("490de21f-db97-49f8-ae2c-48145f592ceb");
+
+    //classification
+    static final UUID classificationUuid = UUID.fromString("25754287-18d8-4352-9cd5-7f2cead4e53c");
+    private static final String classificationName = "German SL";
+
+    //check - import
+    static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
+
+//    static final boolean doTaxa = false;
+//    static final boolean doDeduplicate = true;
+
+    protected void doImport(ICdmDataSource cdmDestination){
+
+        URI source = germanSL();
+
+        //make Source
+        GermanSLImportConfigurator config= GermanSLImportConfigurator.NewInstance(source, cdmDestination);
+        config.setClassificationUuid(classificationUuid);
+        config.setClassificationName(classificationName);
+        config.setCheck(check);
+        config.setDbSchemaValidation(hbm2dll);
+        config.setSecReference(getSecReference());
+        config.setSourceReference(getSourceReference());
+        config.setSource(source);
+        config.setVersionString(versionString);
+
+        String fileName = source.toString();
+        fileName = fileName.substring(fileName.lastIndexOf("/") + 1 );
+
+        String message = "Start import from ("+ fileName + ") ...";
+        System.out.println(message);
+        logger.warn(message);
+
+
+        CdmDefaultImport<GermanSLImportConfigurator> myImport = new CdmDefaultImport<>();
+
+        myImport.invoke(config);
+
+        System.out.println("End import from ("+ source.toString() + ")...");
+
+    }
+
+
+    //GermanSL Excel file
+    public static URI germanSL() {
+        return URI.create("file:////BGBM-PESIHPC/GermanSL/GermanSL.xlsx");
+    }
+
+
+    private Reference getSourceReference() {
+        Reference result = ReferenceFactory.newGeneric();
+        result.setTitle("GermanSL.xslx");
+        Person florian = Person.NewTitledInstance("Jansen");
+        florian.setFirstname("Florian");
+        result.setAuthorship(florian);
+        return result;
+    }
+
+    private Reference getSecReference() {
+        Reference result = ReferenceFactory.newGeneric();
+        result.setTitle("German Standard List");
+//        result.setPlacePublished("Budapest");
+//        result.setPublisher("Akadémiai Kiadó");
+//        result.setPages("608 pp.");
+//        result.setDatePublished(TimePeriodParser.parseString("2012"));
+//        Person borhidi = Person.NewTitledInstance("Borhidi");
+//        borhidi.setFirstname("Attila");
+//        result.setAuthorship(borhidi);
+//        result.setUuid(MexicoConabioTransformer.uuidReferenceBorhidi);
+        return result;
+    }
+
+    /**
+     * @param args
+     */
+    public static void main(String[] args) {
+        GermanSLActivator me = new GermanSLActivator();
+        me.doImport(cdmDestination);
+        System.exit(0);
+    }
+}
+
diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLImportConfigurator.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLImportConfigurator.java
new file mode 100644 (file)
index 0000000..ed29fb0
--- /dev/null
@@ -0,0 +1,73 @@
+/**
+* Copyright (C) 2016 EDIT
+* European Distributed Institute of Taxonomy
+* http://www.e-taxonomy.eu
+*
+* The contents of this file are subject to the Mozilla Public License Version 1.1
+* See LICENSE.TXT at the top of this package for the full license terms.
+*/
+package eu.etaxonomy.cdm.io.redlist.germanSL;
+
+import java.net.URI;
+
+import eu.etaxonomy.cdm.database.ICdmDataSource;
+import eu.etaxonomy.cdm.io.common.ImportStateBase;
+import eu.etaxonomy.cdm.io.common.mapping.IInputTransformer;
+import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
+import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;
+import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
+import eu.etaxonomy.cdm.model.reference.Reference;
+
+/**
+ * @author a.mueller
+ * @date 25.11.2016
+ *
+ */
+public class GermanSLImportConfigurator extends ExcelImportConfiguratorBase{
+    private static final long serialVersionUID = -210709271544496787L;
+
+    private Reference secReference;
+    public Reference getSecReference() { return secReference;}
+    public void setSecReference(Reference secReference) {this.secReference = secReference;}
+
+    private boolean doTaxa = true;
+    public boolean isDoTaxa() {return doTaxa;}
+    public void setDoTaxa(boolean doTaxa) {this.doTaxa = doTaxa;}
+
+    private boolean doCommonNames = true;
+    public boolean isDoCommonNames() {return doCommonNames;}
+    public void setDoCommonNames(boolean doCommonNames) {this.doCommonNames = doCommonNames;}
+
+    private String versionString = "version_1_3_4";
+    public String getVersionString(){return versionString;}
+    public void setVersionString(String versionString){this.versionString = versionString;}
+
+    private static IInputTransformer defaultTransformer = new GermanSLTransformer();
+
+    public static GermanSLImportConfigurator NewInstance(URI source, ICdmDataSource destination) {
+        return new GermanSLImportConfigurator(source, destination);
+    }
+
+    private GermanSLImportConfigurator(URI source, ICdmDataSource destination) {
+        super(source, destination, defaultTransformer);
+        setNomenclaturalCode(NomenclaturalCode.ICNAFP);
+        setSource(source);
+        setDestination(destination);
+     }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    @Override
+    public ImportStateBase getNewState() {
+        return new SimpleExcelTaxonImportState<>(this);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    protected void makeIoClassList() {
+        ioClassList = new Class[]{
+                GermanSLTaxonImport.class,
+                GermanSLTaxonRelationImport.class,
+        };
+    }
+
+}
diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTaxonImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTaxonImport.java
new file mode 100644 (file)
index 0000000..42ae78e
--- /dev/null
@@ -0,0 +1,464 @@
+/**
+* Copyright (C) 2016 EDIT
+* European Distributed Institute of Taxonomy
+* http://www.e-taxonomy.eu
+*
+* The contents of this file are subject to the Mozilla Public License Version 1.1
+* See LICENSE.TXT at the top of this package for the full license terms.
+*/
+package eu.etaxonomy.cdm.io.redlist.germanSL;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import eu.etaxonomy.cdm.common.CdmUtils;
+import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
+import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImport;
+import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;
+import eu.etaxonomy.cdm.model.common.CdmBase;
+import eu.etaxonomy.cdm.model.common.DefinedTerm;
+import eu.etaxonomy.cdm.model.common.Language;
+import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
+import eu.etaxonomy.cdm.model.description.CommonTaxonName;
+import eu.etaxonomy.cdm.model.description.TaxonDescription;
+import eu.etaxonomy.cdm.model.location.Country;
+import eu.etaxonomy.cdm.model.name.BotanicalName;
+import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
+import eu.etaxonomy.cdm.model.name.NonViralName;
+import eu.etaxonomy.cdm.model.name.Rank;
+import eu.etaxonomy.cdm.model.name.RankClass;
+import eu.etaxonomy.cdm.model.name.TaxonNameBase;
+import eu.etaxonomy.cdm.model.reference.Reference;
+import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
+import eu.etaxonomy.cdm.model.taxon.Synonym;
+import eu.etaxonomy.cdm.model.taxon.Taxon;
+import eu.etaxonomy.cdm.model.taxon.TaxonBase;
+
+/**
+ * @author a.mueller
+ * @date 25.11.2016
+ *
+ */
+@Component
+public class GermanSLTaxonImport<CONFIG extends GermanSLImportConfigurator>
+            extends SimpleExcelTaxonImport<CONFIG> {
+
+    private static final long serialVersionUID = 236093186271666895L;
+
+    private static final Logger logger = Logger.getLogger(GermanSLTaxonImport.class);
+
+    static final String SPECIES_NR = "SPECIES_NR";
+    private static final String AUTHOR = "AUTHOR";
+    private static final String ABBREVIAT = "ABBREVIAT";
+    private static final String SEC = "SECUNDUM";
+    private static final String RANG = "RANG";
+    private static final String EXTERNAL_ID = "external_ID";
+    private static final String GRUPPE = "GRUPPE";
+    static final String VALID_NR = "VALID_NR";
+    static final String SYNONYM = "SYNONYM";
+    private static final String NATIVENAME = "NATIVENAME";
+    private static final String LETTER_CODE = "LETTERCODE";
+    static final String AGG = "AGG";
+
+    private static final String AGG_NAME = "AGG_NAME";
+    private static final String VALID_NAME = "VALID_NAME";
+
+    private static final String NACHWEIS = "NACHWEIS";
+    private static final String HYBRID = "HYBRID";
+    private static final String BEGRUEND = "BEGRUEND";
+    private static final String EDITSTATUS = "EDITSTATUS";
+
+    private static final String UUID_ = "UUID";
+
+
+    public static final String TAXON_NAMESPACE = "1.3.4";
+
+    @Override
+    protected String getWorksheetName() {
+        return "1.3.4";
+    }
+
+    //dirty I know, but who cares, needed by distribution and commmon name import
+    protected static final Map<String, TaxonBase<?>> taxonIdMap = new HashMap<>();
+
+
+    private  static List<String> expectedKeys= Arrays.asList(new String[]{
+            SPECIES_NR,EXTERNAL_ID,ABBREVIAT,
+            AUTHOR,SEC,SYNONYM,
+            LETTER_CODE, AGG,
+            NATIVENAME,VALID_NR,RANG,GRUPPE,
+            UUID_,
+            NACHWEIS, HYBRID, BEGRUEND, EDITSTATUS, AGG_NAME, VALID_NAME
+        });
+
+
+    @Override
+    protected void firstPass(SimpleExcelTaxonImportState<CONFIG> state) {
+        String line = state.getCurrentLine() + ": ";
+        HashMap<String, String> record = state.getOriginalRecord();
+
+        Set<String> keys = record.keySet();
+
+        checkAllKeysExist(line, keys, expectedKeys);
+
+        //Name
+        NameResult nameResult = makeName(line, record, state);
+        BotanicalName taxonName = nameResult.name;
+
+      //sec
+        String secRefStr = getValue(record, SEC);
+        Reference sec = getSecRef(state, secRefStr, line);
+
+
+        //status
+        String statusStr = getValue(record, SYNONYM);
+        TaxonBase<?> taxonBase;
+        if (isAccepted(statusStr)){
+            taxonBase = Taxon.NewInstance(taxonName, sec);
+            if (nameResult.proParte){
+                logger.warn(line + "accepted taxon can not be pro parte");
+            }
+        }else{
+            Synonym syn = Synonym.NewInstance(taxonName, sec);
+            if (nameResult.proParte){
+                syn.setProParte(true);
+            }
+            taxonBase = syn;
+        }
+        if (!isBlank(nameResult.sensu)){
+            taxonBase.setAppendedPhrase(nameResult.sensu);
+        }
+        //TODO right order?
+        taxonBase.setAppendedPhrase(CdmUtils.concat(" ", nameResult.auct, taxonBase.getAppendedPhrase()));
+
+        //lettercode
+        String lettercode = getValue(record, LETTER_CODE);
+        if (isNotBlank(lettercode)){
+            UUID idTypeUUID;
+            try {
+                idTypeUUID = state.getTransformer().getIdentifierTypeUuid("LETTERCODE");
+                DefinedTerm idType = getIdentiferType(state, idTypeUUID, "GermanSL lettercode", "GermanSL lettercode", "LETTERCODE", null);
+                taxonBase.addIdentifier(lettercode, idType);
+            } catch (UndefinedTransformerMethodException e) {
+               e.printStackTrace();
+            }
+        }
+
+//        //annotation
+//        String annotation = getValue(record, "Anotacion al Taxon");
+//        if (annotation != null && (!annotation.equals("nom. illeg.") || !annotation.equals("nom. cons."))){
+//            taxonBase.addAnnotation(Annotation.NewInstance(annotation, AnnotationType.EDITORIAL(), Language.SPANISH_CASTILIAN()));
+//        }
+
+        //UUID
+        String uuid = getValue(record, UUID_);
+        //TOOD why sometimes null?
+        if (uuid != null){
+            taxonBase.setUuid(UUID.fromString(uuid));
+        }
+
+
+        //NATIVE NAME
+        String commonNameStr = getValue(record, NATIVENAME);
+        //Ann.: synonym common names should be removed!
+        if (isNotBlank(commonNameStr)){
+            makeCommonName(commonNameStr, taxonBase, line);
+        }
+
+
+        //id
+        String id = getValue(record, SPECIES_NR);
+        this.addOriginalSource(taxonBase, id, TAXON_NAMESPACE, state.getConfig().getSourceReference());
+
+        //save
+//        getTaxonService().save(taxonBase);
+        taxonIdMap.put(id, taxonBase);
+    }
+
+
+    private String removeProparte(String authorStr) {
+        String regEx = "\\s+p\\.\\s*p\\.$";
+        if (authorStr == null || !authorStr.matches(".*" + regEx)){
+            return authorStr;
+        }else{
+            return authorStr.replaceAll(regEx, "");
+        }
+    }
+
+    private String removeSensuLatoStricto(String authorStr) {
+        String regEx = "\\s+s\\.\\s*(l|str)\\.$";
+
+        if (authorStr == null || !authorStr.matches(".*" + regEx)){
+            return authorStr;
+        }else{
+            return authorStr.replaceAll(regEx, "");
+        }
+    }
+
+    private String removeAuct(String authorStr) {
+        String regEx = "auct\\.\\??$";
+
+        if (authorStr == null || !authorStr.matches(/*".*" + */regEx)){
+            return authorStr;
+        }else{
+            return ""; //authorStr.replaceAll(regEx, "");
+        }
+    }
+
+
+    /**
+     * @param state
+     * @param secRefStr
+     * @return
+     */
+    private Reference getSecRef(SimpleExcelTaxonImportState<CONFIG> state, String secRefStr, String line) {
+        Reference result = state.getReference(secRefStr);
+        if (result == null && secRefStr != null){
+            result = ReferenceFactory.newGeneric();
+            result.setTitleCache(secRefStr, true);
+
+//            TimePeriod tp = TimePeriodParser.parseString(secRefStr.substring(secRefStr.length()-4));
+//            String authorStrPart = secRefStr.substring(0, secRefStr.length()-6);
+//            if (! (authorStrPart + ", " + tp.getYear()).equals(secRefStr)){
+//                logger.warn(line + "Sec ref could not be parsed: " + secRefStr);
+//            }else{
+//                result.setDatePublished(tp);
+//            }
+//            TeamOrPersonBase<?> author = state.getAgentBase(authorStrPart);
+//            if (author == null){
+//                if (authorStrPart.contains("&")){
+//                    Team team = Team.NewInstance();
+//                    String[] authorSplit = authorStrPart.split("&");
+//                    String[] firstAuthorSplit = authorSplit[0].trim().split(",");
+//                    for (String authorStr : firstAuthorSplit){
+//                        addTeamMember(team, authorStr);
+//                    }
+//                    addTeamMember(team, authorSplit[1]);
+//                    result.setAuthorship(team);
+//                    state.putAgentBase(team.getTitleCache(), team);
+//                }else if (authorStrPart.equalsIgnoreCase("Tropicos") || authorStrPart.equalsIgnoreCase("The Plant List")
+//                        || authorStrPart.equalsIgnoreCase("APG IV")){
+//                    result.setTitle(authorStrPart);
+//                }else{
+//                    Person person = Person.NewInstance();
+//                    person.setLastname(authorStrPart);
+//                    result.setAuthorship(person);
+//                    state.putAgentBase(person.getTitleCache(), person);
+//                }
+//            }else{
+//                result.setAuthorship(author);
+//            }
+            state.putReference(secRefStr, result);
+        }
+
+        return result;
+    }
+
+
+    private class NameResult{
+        BotanicalName name;
+        boolean proParte = false;
+        String sensu = null;
+        String auct = null;
+    }
+
+    /**
+     * @param record
+     * @param state
+     * @return
+     */
+    private NameResult makeName(String line, HashMap<String, String> record, SimpleExcelTaxonImportState<CONFIG> state) {
+
+        String specieNrStr = getValue(record, SPECIES_NR);
+        String nameStr = getValue(record, ABBREVIAT);
+        String authorStr = getValue(record, AUTHOR);
+        String rankStr = getValue(record, RANG);
+
+        NameResult result = new NameResult();
+
+        //rank
+        Rank rank = makeRank(line, state, rankStr);
+
+        //name
+        nameStr = normalizeNameStr(nameStr);
+        String nameStrWithoutSensu = removeSensuLatoStricto(nameStr);
+        if (nameStrWithoutSensu.length() < nameStr.length()){
+            result.sensu = nameStr.substring(nameStrWithoutSensu.length()).trim();
+            nameStr = nameStrWithoutSensu;
+        }
+
+        //author
+        //pp
+        authorStr = normalizeAuthorStr(authorStr);
+        String authorStrWithoutProParte = removeProparte(authorStr);
+        result.proParte = authorStrWithoutProParte.length() < authorStr.length();
+        authorStr = authorStrWithoutProParte;
+
+        //auct.
+        String authorStrWithoutAuct = removeAuct(authorStr);
+        if (authorStrWithoutAuct.length() < authorStr.length()){
+            result.auct = authorStr.substring(authorStrWithoutAuct.length()).trim();
+        }
+        authorStr = authorStrWithoutAuct;
+
+
+        //name+author
+        String fullNameStr = CdmUtils.concat(" ", nameStr, authorStr);
+
+        BotanicalName fullName = (BotanicalName)nameParser.parseReferencedName(fullNameStr, NomenclaturalCode.ICNAFP, rank);
+        if (fullName.isProtectedTitleCache()){
+            logger.warn(line + "Name could not be parsed: " + fullNameStr );
+        }else{
+            replaceAuthorNamesAndNomRef(state, fullName);
+        }
+//        BotanicalName existingName = getExistingName(state, fullName);
+
+        //TODO handle existing name
+        BotanicalName name = fullName;
+        this.addOriginalSource(name, specieNrStr, TAXON_NAMESPACE + "_Name", state.getConfig().getSourceReference());
+
+        result.name = name;
+        return result;
+    }
+
+
+
+    /**
+     * @param line
+     * @param state
+     * @param rankStr
+     * @return
+     */
+    private Rank makeRank(String line, SimpleExcelTaxonImportState<CONFIG> state, String rankStr) {
+        Rank rank = null;
+        try {
+            rank = state.getTransformer().getRankByKey(rankStr);
+            if (rank == null){
+                UUID rankUuid = state.getTransformer().getRankUuid(rankStr);
+                OrderedTermVocabulary<Rank> voc = (OrderedTermVocabulary<Rank>)Rank.SPECIES().getVocabulary();
+                //TODO
+                Rank lowerRank = Rank.FORM();
+                rank = getRank(state, rankUuid, rankStr, rankStr, rankStr, voc, lowerRank, RankClass.Infraspecific);
+                if (rank == null){
+                    logger.warn(line + "Rank not recognized: " + rankStr);
+                }
+            }
+        } catch (Exception e1) {
+                logger.warn(line + "Rank not recognized: " + rankStr);
+        }
+        return rank;
+    }
+
+
+    /**
+     * @param authorStr
+     * @return
+     */
+    private String normalizeAuthorStr(String authorStr) {
+        if (isBlank(authorStr)){
+            return "";
+        }else{
+            if (authorStr.equals("-") || authorStr.equals("#")){
+                authorStr = "";
+            }
+            return authorStr;
+        }
+    }
+
+    private String normalizeNameStr(String nameStr) {
+        nameStr = nameStr
+                .replace(" agg.", " aggr.")
+                .replace(" fo. ", " f. ")
+             ;
+        return nameStr;
+    }
+
+
+    boolean nameMapIsInitialized = false;
+    /**
+     * @param state
+     * @param fullName
+     * @return
+     */
+    private BotanicalName getExistingName(SimpleExcelTaxonImportState<CONFIG> state, BotanicalName fullName) {
+        initExistinNames(state);
+        return (BotanicalName)state.getName(fullName.getTitleCache());
+    }
+
+    /**
+     * @param state
+     */
+    @SuppressWarnings("rawtypes")
+    private void initExistinNames(SimpleExcelTaxonImportState<CONFIG> state) {
+        if (!nameMapIsInitialized){
+            List<String> propertyPaths = Arrays.asList("");
+            List<TaxonNameBase> existingNames = this.getNameService().list(null, null, null, null, propertyPaths);
+            for (TaxonNameBase tnb : existingNames){
+                state.putName(tnb.getTitleCache(), (NonViralName<?>)tnb);
+            }
+            nameMapIsInitialized = true;
+        }
+    }
+
+
+    /**
+     * @param commmonNameStr
+     * @param taxonBase
+     */
+    private void makeCommonName(String commmonNameStr, TaxonBase<?> taxonBase, String line) {
+        if (taxonBase.isInstanceOf(Synonym.class)){
+            //synonym common names should be neglected
+            return;
+        }
+        Taxon acceptedTaxon = getAccepted(taxonBase);
+        if (acceptedTaxon != null){
+            TaxonDescription desc = getTaxonDescription(acceptedTaxon, false, true);
+            desc.setDefault(true);
+            CommonTaxonName commonName = CommonTaxonName.NewInstance(commmonNameStr, Language.GERMAN(), Country.GERMANY());
+            desc.addElement(commonName);
+        }else{
+            logger.warn(line + "No accepted taxon available");
+        }
+
+    }
+
+
+
+    private boolean isAccepted(String statusStr){
+        if ("FALSE()".equals(statusStr) || "0".equals(statusStr) || "false".equalsIgnoreCase(statusStr)){
+            return true;
+        } else if ("TRUE()".equals(statusStr) || "1".equals(statusStr)|| "true".equalsIgnoreCase(statusStr)){
+            return false;
+        }else{
+            logger.warn("Unhandled taxon status: " + statusStr);
+            return false;
+        }
+    }
+
+
+    /**
+     * @param next
+     * @return
+     */
+    private Taxon getAccepted(TaxonBase<?> taxonBase) {
+        if (taxonBase.isInstanceOf(Taxon.class)){
+            return CdmBase.deproxy(taxonBase, Taxon.class);
+        }else{
+            Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
+            return syn.getAcceptedTaxon();
+        }
+    }
+
+
+    @Override
+    protected boolean isIgnore(SimpleExcelTaxonImportState<CONFIG> state) {
+        return ! state.getConfig().isDoTaxa();
+    }
+}
diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTaxonRelationImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTaxonRelationImport.java
new file mode 100644 (file)
index 0000000..9060f07
--- /dev/null
@@ -0,0 +1,160 @@
+/**
+* Copyright (C) 2016 EDIT
+* European Distributed Institute of Taxonomy
+* http://www.e-taxonomy.eu
+*
+* The contents of this file are subject to the Mozilla Public License Version 1.1
+* See LICENSE.TXT at the top of this package for the full license terms.
+*/
+package eu.etaxonomy.cdm.io.redlist.germanSL;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImport;
+import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;
+import eu.etaxonomy.cdm.model.common.CdmBase;
+import eu.etaxonomy.cdm.model.reference.Reference;
+import eu.etaxonomy.cdm.model.taxon.Classification;
+import eu.etaxonomy.cdm.model.taxon.Synonym;
+import eu.etaxonomy.cdm.model.taxon.SynonymType;
+import eu.etaxonomy.cdm.model.taxon.Taxon;
+import eu.etaxonomy.cdm.model.taxon.TaxonBase;
+
+/**
+ * @author a.mueller
+ * @date 25.11.2016
+ *
+ */
+@Component
+public class GermanSLTaxonRelationImport<CONFIG extends GermanSLImportConfigurator>
+            extends SimpleExcelTaxonImport<CONFIG> {
+
+    private static final long serialVersionUID = 3381597141845204995L;
+
+    private static final Logger logger = Logger.getLogger(GermanSLTaxonRelationImport.class);
+
+    public static final String TAXON_NAMESPACE = "1.3.4";
+
+    @Override
+    protected String getWorksheetName() {
+        return "1.3.4";
+    }
+
+    private Classification classification;
+    private Set<TaxonBase> taxaToSave = new HashSet<>();
+    private int count = 0;
+
+    @Override
+    protected void firstPass(SimpleExcelTaxonImportState<CONFIG> state) {
+        count++;
+        Map<String, String> record = state.getOriginalRecord();
+        String line = state.getCurrentLine() + ": ";
+
+        String parentStr = getValue(record, GermanSLTaxonImport.AGG);
+        String acceptedStr = getValue(record, GermanSLTaxonImport.VALID_NR);
+        String idStr = getValue(record, GermanSLTaxonImport.SPECIES_NR);
+        String statusStr = getValue(record, GermanSLTaxonImport.SYNONYM);
+
+        Classification classification = getClassification(state);
+        TaxonBase<?> taxonBase = GermanSLTaxonImport.taxonIdMap.get(idStr);
+        Taxon parent;
+        if (isAccepted(statusStr)){
+            TaxonBase<?> parentTmp = GermanSLTaxonImport.taxonIdMap.get(parentStr);
+            if (parentTmp == null){
+                logger.warn(line + "Parent is missing: "+ parentStr);
+            }else if (parentTmp.isInstanceOf(Synonym.class)){
+                logger.warn(line + "Parent is not of type Taxon");
+            }else{
+                parent = (Taxon)parentTmp;
+                Taxon taxon = (Taxon)taxonBase;
+                Reference relRef = null;  //TODO
+                if ("0".equals(idStr)){
+                    classification.addChildTaxon(taxon, relRef, null);
+                }else{
+                    classification.addParentChild(parent, taxon, relRef, null);
+                }
+            }
+        } else {
+            TaxonBase<?> parentTmp = GermanSLTaxonImport.taxonIdMap.get(acceptedStr);
+            if (parentTmp == null){
+                logger.warn(line + "Accepted taxon is missing: " + acceptedStr);
+            }else if (parentTmp.isInstanceOf(Synonym.class)){
+                logger.warn(line + "Accepted taxon is not of type Taxon");
+            }else{
+                parent = (Taxon)parentTmp;
+                Synonym synonym = (Synonym)taxonBase;
+                parent.addSynonym(synonym, SynonymType.SYNONYM_OF());
+            }
+        }
+        taxaToSave.add(taxonBase);
+        if ((count % 1000) == 0){
+            count = 0;
+            getTaxonService().saveOrUpdate(taxaToSave);
+            taxaToSave = new HashSet<>();
+        }
+    }
+
+
+    private boolean isAccepted(String statusStr){
+        if ("FALSE()".equals(statusStr) || "0".equals(statusStr) || "false".equalsIgnoreCase(statusStr)){
+            return true;
+        } else if ("TRUE()".equals(statusStr) || "1".equals(statusStr)|| "true".equalsIgnoreCase(statusStr)){
+            return false;
+        }else{
+            logger.warn("Unhandled taxon status: " + statusStr);
+            return false;
+        }
+    }
+
+
+    /**
+     * @param next
+     * @return
+     */
+    private Taxon getAccepted(TaxonBase<?> taxonBase) {
+        if (taxonBase.isInstanceOf(Taxon.class)){
+            return CdmBase.deproxy(taxonBase, Taxon.class);
+        }else{
+            Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
+            return syn.getAcceptedTaxon();
+        }
+    }
+
+
+    boolean needsFinalSave = true;
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    protected void secondPass(SimpleExcelTaxonImportState<CONFIG> state) {
+        if (needsFinalSave){
+            getTaxonService().saveOrUpdate(taxaToSave);
+            needsFinalSave = false;
+        }
+    }
+
+
+    /**
+     * @return
+     */
+    private Classification getClassification(SimpleExcelTaxonImportState<CONFIG> state) {
+        if (classification == null){
+            GermanSLImportConfigurator config = state.getConfig();
+            classification = Classification.NewInstance(config.getClassificationName());
+            classification.setUuid(config.getClassificationUuid());
+            classification.setReference(config.getSecReference());
+            getClassificationService().save(classification);
+        }
+        return classification;
+    }
+
+    @Override
+    protected boolean isIgnore(SimpleExcelTaxonImportState<CONFIG> state) {
+        return ! state.getConfig().isDoTaxa();
+    }
+}
diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTransformer.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/germanSL/GermanSLTransformer.java
new file mode 100644 (file)
index 0000000..cdcfe59
--- /dev/null
@@ -0,0 +1,93 @@
+/**
+* Copyright (C) 2016 EDIT
+* European Distributed Institute of Taxonomy
+* http://www.e-taxonomy.eu
+*
+* The contents of this file are subject to the Mozilla Public License Version 1.1
+* See LICENSE.TXT at the top of this package for the full license terms.
+*/
+package eu.etaxonomy.cdm.io.redlist.germanSL;
+
+import java.util.UUID;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+
+import eu.etaxonomy.cdm.io.common.mapping.InputTransformerBase;
+import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
+import eu.etaxonomy.cdm.model.name.Rank;
+
+/**
+ * @author a.mueller
+ * @date 25.11.2016
+ *
+ */
+public class GermanSLTransformer  extends InputTransformerBase{
+
+    private static final long serialVersionUID = -1794363151658943665L;
+
+    @SuppressWarnings("unused")
+    private static final Logger logger = Logger.getLogger(GermanSLTransformer.class);
+
+    private static final UUID uuidRankZUS = UUID.fromString("db161a9e-aebc-419f-a724-14feae1b08f7");
+    private static final UUID uuidIdentifierTypeLettercode = UUID.fromString("99b907df-c932-4007-96e9-b6a0d5e1f3bf");
+
+
+    @Override
+    public UUID getRankUuid(String key) throws UndefinedTransformerMethodException {
+        if (StringUtils.isBlank(key)){
+            return null;
+        }else if (key.equals("ZUS")) {return uuidRankZUS;
+        }else{
+            return null;
+        }
+    }
+
+
+    @Override
+    public Rank getRankByKey(String key) throws UndefinedTransformerMethodException {
+        if (StringUtils.isBlank(key)){
+            return null;
+        }else if (key.equals("ABT")) {return Rank.DIVISION();
+        }else if (key.equals("AG2")){return Rank.INFRAGENERICTAXON(); //Aggregate oberhalb der Gattung (Auswertungsgruppe)
+        }else if (key.equals("AGG")){return Rank.SPECIESAGGREGATE();
+        }else if (key.equals("FAM")){return Rank.FAMILY();
+        }else if (key.equals("FOR")){return Rank.FORM();
+        }else if (key.equals("GAT")){return Rank.GENUS();
+        }else if (key.equals("KLA")){return Rank.CLASS();
+        }else if (key.equals("ORD")){return Rank.ORDER();
+//        }else if (key.equals("ROOT")){return Rank.ROOT();
+        }else if (key.equals("SEC")){return Rank.SECTION_BOTANY();
+        }else if (key.equals("SER")){return Rank.SERIES();
+
+        }else if (key.equals("SGE")){return Rank.SUBGENUS();
+//        }else if (key.equals("SGR")){return Rank.();
+        }else if (key.equals("SPE")){return Rank.SPECIES();
+        }else if (key.equals("SSE")){return Rank.SUBSERIES();
+        }else if (key.equals("SSP")){return Rank.SUBSPECIES();
+        }else if (key.equals("UAB")){return Rank.SUBDIVISION();
+        }else if (key.equals("UKL")){return Rank.SUBCLASS();
+        }else if (key.equals("VAR")){return Rank.VARIETY();
+//        }else if (key.equals("ZUS")){return Rank.();
+        }else{
+            return null;
+        }
+    }
+
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public UUID getIdentifierTypeUuid(String key) throws UndefinedTransformerMethodException {
+        if (StringUtils.isBlank(key)){
+            return null;
+        }else if (key.equals("LETTERCODE")) {return uuidIdentifierTypeLettercode;
+        }else{
+            return null;
+        }
+    }
+
+
+
+}