smaller changes in specimen import and excel import
authorKatja Luther <k.luther@bgbm.org>
Mon, 19 Jun 2017 13:23:20 +0000 (15:23 +0200)
committerKatja Luther <k.luther@bgbm.org>
Mon, 19 Jun 2017 13:23:20 +0000 (15:23 +0200)
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImportConfiguratorBase.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImporterBase.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImport.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImportConfigurator.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/NormalExplicitImport.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/TaxonExcelImportState.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportStateBase.java

index e42b3d6b8d53ce1d0944a6a79be37b30d8835cad..db5c3a92da688719d983beacc23804a7dfb8b822 100755 (executable)
@@ -29,7 +29,8 @@ public abstract class ExcelImportConfiguratorBase
        //TODO\r
        private static IInputTransformer defaultTransformer = null;\r
        private byte[] stream;\r
-\r
+       private boolean deduplicateReferences = true;\r
+       private boolean deduplicateAuthors = false;\r
        /**\r
         * @param url\r
         * @param destination\r
@@ -70,4 +71,26 @@ public abstract class ExcelImportConfiguratorBase
            public void setStream(byte[] stream) {\r
                this.stream = stream;\r
            }\r
+\r
+        /**\r
+         * @return the deduplicateReferences\r
+         */\r
+        public boolean isDeduplicateReferences() {\r
+            return deduplicateReferences;\r
+        }\r
+\r
+        /**\r
+         * @param deduplicateReferences the deduplicateReferences to set\r
+         */\r
+        public void setDeduplicateReferences(boolean deduplicateReferences) {\r
+            this.deduplicateReferences = deduplicateReferences;\r
+        }\r
+\r
+        public boolean isDeduplicateAuthors() {\r
+            return deduplicateAuthors;\r
+        }\r
+\r
+        public void setDeduplicateAuthors(boolean deduplicateAuthors) {\r
+            this.deduplicateAuthors = deduplicateAuthors;\r
+        }\r
 }\r
index 3f3f1b1e63fc99f6481cd23ed90760174966037b..63d04079b905e5f39ecffdc9b48d0f312f5f6ce1 100755 (executable)
@@ -25,8 +25,10 @@ import eu.etaxonomy.cdm.common.ExcelUtils;
 import eu.etaxonomy.cdm.io.common.CdmImportBase;
 import eu.etaxonomy.cdm.io.distribution.excelupdate.ExcelDistributionUpdateConfigurator;
 import eu.etaxonomy.cdm.io.excel.taxa.NormalExplicitImportConfigurator;
+import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
 import eu.etaxonomy.cdm.model.common.TimePeriod;
 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
+import eu.etaxonomy.cdm.model.reference.Reference;
 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
 
 /**
@@ -144,7 +146,12 @@ public abstract class ExcelImporterBase<STATE extends ExcelImportState<? extends
                 secondPass(state);
                        state.incCurrentLine();
                }
-
+               if (configurator.isDeduplicateReferences()){
+                   getReferenceService().deduplicate(Reference.class, null, null);
+               }
+               if (configurator.isDeduplicateAuthors()){
+                getAgentService().deduplicate(TeamOrPersonBase.class, null, null);
+            }
                commitTransaction(txStatus);
        }else{
                logger.warn("No records found in " + source);
index 0c260622140a050c6437c29999a015830f2fd913..28515fe09435ad7bd8b074137468ec3c49142d61 100644 (file)
@@ -26,9 +26,9 @@ import eu.etaxonomy.cdm.common.CdmUtils;
 import eu.etaxonomy.cdm.common.ExcelUtils;\r
 import eu.etaxonomy.cdm.io.common.CdmImportBase;\r
 import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;\r
-import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;\r
 import eu.etaxonomy.cdm.io.excel.common.ExcelImportState;\r
 import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase;\r
+import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;\r
 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;\r
 import eu.etaxonomy.cdm.model.description.Distribution;\r
 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;\r
index 685288d7ea3b02b21ef481766756d35e5796f045..44d899c3867bf4473a4a7351d4b0239935c261e5 100644 (file)
@@ -14,8 +14,8 @@ import org.apache.log4j.Logger;
 \r
 import eu.etaxonomy.cdm.database.ICdmDataSource;\r
 import eu.etaxonomy.cdm.io.common.IImportConfigurator;\r
-import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;\r
 import eu.etaxonomy.cdm.io.excel.common.ExcelImportState;\r
+import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;\r
 import eu.etaxonomy.cdm.model.reference.Reference;\r
 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;\r
 \r
index d91b158a4734f01c54208c7b320c65d5ab2497a0..15a6e58a7bb6420cd4b8f4860a0abbaf5aed59ca 100644 (file)
@@ -285,11 +285,12 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
                //protologue
                for (String protologue : taxonDataHolder.getProtologues()){
                        TextData textData = TextData.NewInstance(Feature.PROTOLOGUE());
-                       this.getNameDescription(taxonBase.getName()).addElement(textData);
+                       this.getNameDescription(taxonBase.getName(), state).addElement(textData);
                        URI uri;
                        try {
                                uri = new URI(protologue);
                                textData.addMedia(Media.NewInstance(uri, null, null, null));
+
                        } catch (URISyntaxException e) {
                                String warning = "URISyntaxException when trying to convert to URI: " + protologue;
                                logger.error(warning);
@@ -379,10 +380,10 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
                                                        if (parentTaxon != null) {
                                                                //Taxon taxon = (Taxon)state.getTaxonBase(childId);
 
-                                                               Reference sourceRef = state.getConfig().getSourceReference();
+                                                       //      Reference sourceRef = state.getConfig().getSourceReference();
                                                                String microCitation = null;
                                                                Taxon childTaxon = taxon;
-                                                               makeParent(state, parentTaxon, childTaxon, sourceRef, microCitation);
+                                                               makeParent(state, parentTaxon, childTaxon, null, microCitation);
                                                                getTaxonService().saveOrUpdate(childTaxon);
                                                                state.putTaxon(parentId, parentTaxon);
                                                        } else {
@@ -420,10 +421,10 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
                                            if (parentTaxon != null) {
                                 Taxon taxon = (Taxon)state.getTaxonBase(childId);
 
-                                Reference sourceRef = state.getConfig().getSourceReference();
+                               // Reference sourceRef = state.getConfig().getSourceReference();
                                 String microCitation = null;
                                 Taxon childTaxon = taxon;
-                                makeParent(state, parentTaxon, childTaxon, sourceRef, microCitation);
+                                makeParent(state, parentTaxon, childTaxon, null, microCitation);
                                 getTaxonService().saveOrUpdate(parentTaxon);
                                 state.putTaxon(parentId, parentTaxon);
                             } else {
@@ -790,15 +791,15 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
                        if (StringUtils.isNotBlank(reference)) {
                            String pub = CdmUtils.concat(" ", reference, state.getCurrentRow().getCollation());
                            String[] split = pub.split(":");
-
-                           INomenclaturalReference ref = parser.parseReferenceTitle(reference, date, true);
+                           pub = split[0];
+                           INomenclaturalReference ref = parser.parseReferenceTitle(pub, date, true);
                            if (split.length > 1){
                     String detail = split[split.length-1];
                     taxonName.setNomenclaturalMicroReference(detail.trim());
-                    pub = pub.substring(0, pub.length() - detail.length() - 1).trim();
+
                 }
 
-                ref.setAbbrevTitle(pub);
+             //   ref.setAbbrevTitle(pub);
 
 
                            if (ref.getAuthorship() == null){
@@ -817,15 +818,15 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
                }
 
                //Create the taxon
-               Reference sec = state.getConfig().getSourceReference();
+               //Reference sec = state.getConfig().getSourceReference();
                // Create the status
                nameStatus = CdmUtils.Nz(nameStatus).trim().toLowerCase();
                if (validMarkers.contains(nameStatus)){
-                       taxonBase = Taxon.NewInstance(taxonName, sec);
+                       taxonBase = Taxon.NewInstance(taxonName, null);
                }else if (synonymMarkers.contains(nameStatus)){
-                       taxonBase = Synonym.NewInstance(taxonName, sec);
+                       taxonBase = Synonym.NewInstance(taxonName, null);
                }else {
-                       Taxon taxon = Taxon.NewInstance(taxonName, sec);
+                       Taxon taxon = Taxon.NewInstance(taxonName, null);
                        if (nameStatusMarkers.contains(nameStatus)){
                            if (nameStatus.equals(NOM_ILLEG)){
                                taxonName.addStatus(NomenclaturalStatusType.ILLEGITIMATE(), null, null);
@@ -839,8 +840,8 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
                        }
                        taxonBase = taxon;
                }
-               taxonBase.getName().addSource(OriginalSourceType.Import, null,"TaxonName" ,null, null);
-               taxonBase.addSource(OriginalSourceType.Import, null,"TaxonName" ,null, null);
+               taxonBase.getName().addSource(OriginalSourceType.Import, null,"TaxonName" ,state.getConfig().getSourceReference(), null);
+               taxonBase.addSource(OriginalSourceType.Import, null,"TaxonName" ,state.getConfig().getSourceReference(), null);
 
                return taxonBase;
        }
@@ -852,14 +853,16 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
        //TODO implementation must be improved when matching of taxon names with existing names is implemented
        //=> the assumption that the only description is the description added by this import
        //is wrong then
-       private TaxonNameDescription getNameDescription(TaxonName name) {
+       private TaxonNameDescription getNameDescription(TaxonName name, TaxonExcelImportState state) {
                Set<TaxonNameDescription> descriptions = name.getDescriptions();
                if (descriptions.size()>1){
                        throw new IllegalStateException("Implementation does not yet support names with multiple descriptions");
                }else if (descriptions.size()==1){
                        return descriptions.iterator().next();
                }else{
-                       return TaxonNameDescription.NewInstance(name);
+                   TaxonNameDescription desc = TaxonNameDescription.NewInstance(name);
+                   desc.addSource(OriginalSourceType.Import, null, "NameDescription", state.getConfig().getSourceReference(), null);
+                       return desc;
                }
        }
 
index 8d13b78b508619d2b8e120649117ddfdcf5e0913..c4da42f997520793cf60874232cde337e3501a39 100644 (file)
@@ -16,8 +16,8 @@ import java.util.Set;
 \r
 import org.apache.log4j.Logger;\r
 \r
-import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;\r
 import eu.etaxonomy.cdm.io.excel.common.ExcelImportState;\r
+import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;\r
 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;\r
 import eu.etaxonomy.cdm.model.taxon.Classification;\r
 import eu.etaxonomy.cdm.model.taxon.Taxon;\r
index db92c0f27f75e96ca859e08124b619d9f2d86c4a..ef782c0cb272c3fad8843e84b074705a0cbe0c80 100755 (executable)
@@ -553,7 +553,9 @@ public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STA
                if (cdmBase.isInstanceOf(LanguageString.class)){
                    return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
                }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
-                   return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
+                   SpecimenOrObservationBase specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
+
+                   return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
                }else if (cdmBase.isInstanceOf(Reference.class)){
                    return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
                }else if (cdmBase.isInstanceOf(Classification.class)){
@@ -563,7 +565,9 @@ public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STA
                }else if (cdmBase.isInstanceOf(Collection.class)){
                    return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
                }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
-                   return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
+                   DescriptionBase description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
+
+                   return cdmRepository.getDescriptionService().saveOrUpdate(description);
                }else if (cdmBase.isInstanceOf(TaxonBase.class)){
                    return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
                }else if (cdmBase.isInstanceOf(TaxonName.class)){
@@ -1008,14 +1012,14 @@ public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STA
                if (!existsInClassification(taxon,state.getClassification(), state)){
                    if(config.isMoveNewTaxaToDefaultClassification()){
                        //check if node exists in default classification
-                       if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
-                           addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
+                       if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
+                           addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
                        }
-                   }
-                   else {
+                   }else{
                        //add non-existing taxon to current classification
                        addParentTaxon(taxon, state, preferredFlag, state.getClassification());
                    }
+
                }
            }
 
index e1bcd519a3fac199ab279af5a7205ec0f0c26179..5145dd1eaf23a5b9ab71f95fcf38e30cc0e7b456 100644 (file)
@@ -189,8 +189,8 @@ public class SpecimenImportStateBase<CONFIG extends SpecimenImportConfiguratorBa
         this.classification = classification;
     }
 
-    public Classification getDefaultClassification() {
-        if(defaultClassification==null){
+    public Classification getDefaultClassification(boolean createIfNotExist) {
+        if(defaultClassification==null && createIfNotExist){
             final String defaultClassificationAbcd = "Default Classification Specimen Import";
             for (Classification classif : cdmRepository.getClassificationService().list(Classification.class, null, null, null, null)){
                 if (classif.getTitleCache()!=null && classif.getTitleCache().equalsIgnoreCase(defaultClassificationAbcd)