From: Katja Luther <k.luther@bgbm.org>
Date: Mon, 19 Jun 2017 13:23:20 +0000 (+0200)
Subject: smaller changes in specimen import and excel import
X-Git-Tag: 4.8.0^2~74
X-Git-Url: https://dev.e-taxonomy.eu/gitweb/cdmlib.git/commitdiff_plain/451fef0fce4ca56943497843dd27b22dfdfd9530

smaller changes in specimen import and excel import
---

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImportConfiguratorBase.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImportConfiguratorBase.java
index e42b3d6b8d..db5c3a92da 100755
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImportConfiguratorBase.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImportConfiguratorBase.java
@@ -29,7 +29,8 @@ public abstract class ExcelImportConfiguratorBase
 	//TODO
 	private static IInputTransformer defaultTransformer = null;
 	private byte[] stream;
-
+	private boolean deduplicateReferences = true;
+	private boolean deduplicateAuthors = false;
 	/**
 	 * @param url
 	 * @param destination
@@ -70,4 +71,26 @@ public abstract class ExcelImportConfiguratorBase
 	    public void setStream(byte[] stream) {
 	        this.stream = stream;
 	    }
+
+        /**
+         * @return the deduplicateReferences
+         */
+        public boolean isDeduplicateReferences() {
+            return deduplicateReferences;
+        }
+
+        /**
+         * @param deduplicateReferences the deduplicateReferences to set
+         */
+        public void setDeduplicateReferences(boolean deduplicateReferences) {
+            this.deduplicateReferences = deduplicateReferences;
+        }
+
+        public boolean isDeduplicateAuthors() {
+            return deduplicateAuthors;
+        }
+
+        public void setDeduplicateAuthors(boolean deduplicateAuthors) {
+            this.deduplicateAuthors = deduplicateAuthors;
+        }
 }
diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImporterBase.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImporterBase.java
index 3f3f1b1e63..63d04079b9 100755
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImporterBase.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelImporterBase.java
@@ -25,8 +25,10 @@ import eu.etaxonomy.cdm.common.ExcelUtils;
 import eu.etaxonomy.cdm.io.common.CdmImportBase;
 import eu.etaxonomy.cdm.io.distribution.excelupdate.ExcelDistributionUpdateConfigurator;
 import eu.etaxonomy.cdm.io.excel.taxa.NormalExplicitImportConfigurator;
+import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
 import eu.etaxonomy.cdm.model.common.TimePeriod;
 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
+import eu.etaxonomy.cdm.model.reference.Reference;
 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
 
 /**
@@ -144,7 +146,12 @@ public abstract class ExcelImporterBase<STATE extends ExcelImportState<? extends
                 secondPass(state);
     			state.incCurrentLine();
     	   	}
-
+    		if (configurator.isDeduplicateReferences()){
+    		    getReferenceService().deduplicate(Reference.class, null, null);
+    		}
+    		if (configurator.isDeduplicateAuthors()){
+                getAgentService().deduplicate(TeamOrPersonBase.class, null, null);
+            }
     		commitTransaction(txStatus);
     	}else{
     		logger.warn("No records found in " + source);
diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImport.java
index 0c26062214..28515fe094 100644
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImport.java
@@ -26,9 +26,9 @@ import eu.etaxonomy.cdm.common.CdmUtils;
 import eu.etaxonomy.cdm.common.ExcelUtils;
 import eu.etaxonomy.cdm.io.common.CdmImportBase;
 import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;
-import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
 import eu.etaxonomy.cdm.io.excel.common.ExcelImportState;
 import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase;
+import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
 import eu.etaxonomy.cdm.model.description.Distribution;
 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImportConfigurator.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImportConfigurator.java
index 685288d7ea..44d899c386 100644
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImportConfigurator.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/distribution/DistributionImportConfigurator.java
@@ -14,8 +14,8 @@ import org.apache.log4j.Logger;
 
 import eu.etaxonomy.cdm.database.ICdmDataSource;
 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
-import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
 import eu.etaxonomy.cdm.io.excel.common.ExcelImportState;
+import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
 import eu.etaxonomy.cdm.model.reference.Reference;
 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
 
diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/NormalExplicitImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/NormalExplicitImport.java
index d91b158a47..15a6e58a7b 100644
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/NormalExplicitImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/NormalExplicitImport.java
@@ -285,11 +285,12 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
 		//protologue
 		for (String protologue : taxonDataHolder.getProtologues()){
 			TextData textData = TextData.NewInstance(Feature.PROTOLOGUE());
-			this.getNameDescription(taxonBase.getName()).addElement(textData);
+			this.getNameDescription(taxonBase.getName(), state).addElement(textData);
 			URI uri;
 			try {
 				uri = new URI(protologue);
 				textData.addMedia(Media.NewInstance(uri, null, null, null));
+
 			} catch (URISyntaxException e) {
 				String warning = "URISyntaxException when trying to convert to URI: " + protologue;
 				logger.error(warning);
@@ -379,10 +380,10 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
     							if (parentTaxon != null) {
     								//Taxon taxon = (Taxon)state.getTaxonBase(childId);
 
-    								Reference sourceRef = state.getConfig().getSourceReference();
+    							//	Reference sourceRef = state.getConfig().getSourceReference();
     								String microCitation = null;
     								Taxon childTaxon = taxon;
-    								makeParent(state, parentTaxon, childTaxon, sourceRef, microCitation);
+    								makeParent(state, parentTaxon, childTaxon, null, microCitation);
     								getTaxonService().saveOrUpdate(childTaxon);
     								state.putTaxon(parentId, parentTaxon);
     							} else {
@@ -420,10 +421,10 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
     					    if (parentTaxon != null) {
                                 Taxon taxon = (Taxon)state.getTaxonBase(childId);
 
-                                Reference sourceRef = state.getConfig().getSourceReference();
+                               // Reference sourceRef = state.getConfig().getSourceReference();
                                 String microCitation = null;
                                 Taxon childTaxon = taxon;
-                                makeParent(state, parentTaxon, childTaxon, sourceRef, microCitation);
+                                makeParent(state, parentTaxon, childTaxon, null, microCitation);
                                 getTaxonService().saveOrUpdate(parentTaxon);
                                 state.putTaxon(parentId, parentTaxon);
                             } else {
@@ -790,15 +791,15 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
 			if (StringUtils.isNotBlank(reference)) {
 			    String pub = CdmUtils.concat(" ", reference, state.getCurrentRow().getCollation());
 			    String[] split = pub.split(":");
-
-			    INomenclaturalReference ref = parser.parseReferenceTitle(reference, date, true);
+			    pub = split[0];
+			    INomenclaturalReference ref = parser.parseReferenceTitle(pub, date, true);
 			    if (split.length > 1){
                     String detail = split[split.length-1];
                     taxonName.setNomenclaturalMicroReference(detail.trim());
-                    pub = pub.substring(0, pub.length() - detail.length() - 1).trim();
+
                 }
 
-                ref.setAbbrevTitle(pub);
+             //   ref.setAbbrevTitle(pub);
 
 
 			    if (ref.getAuthorship() == null){
@@ -817,15 +818,15 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
 		}
 
 		//Create the taxon
-		Reference sec = state.getConfig().getSourceReference();
+		//Reference sec = state.getConfig().getSourceReference();
 		// Create the status
 		nameStatus = CdmUtils.Nz(nameStatus).trim().toLowerCase();
 		if (validMarkers.contains(nameStatus)){
-			taxonBase = Taxon.NewInstance(taxonName, sec);
+			taxonBase = Taxon.NewInstance(taxonName, null);
 		}else if (synonymMarkers.contains(nameStatus)){
-			taxonBase = Synonym.NewInstance(taxonName, sec);
+			taxonBase = Synonym.NewInstance(taxonName, null);
 		}else {
-			Taxon taxon = Taxon.NewInstance(taxonName, sec);
+			Taxon taxon = Taxon.NewInstance(taxonName, null);
 			if (nameStatusMarkers.contains(nameStatus)){
 			    if (nameStatus.equals(NOM_ILLEG)){
 			        taxonName.addStatus(NomenclaturalStatusType.ILLEGITIMATE(), null, null);
@@ -839,8 +840,8 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
 			}
 			taxonBase = taxon;
 		}
-		taxonBase.getName().addSource(OriginalSourceType.Import, null,"TaxonName" ,null, null);
-		taxonBase.addSource(OriginalSourceType.Import, null,"TaxonName" ,null, null);
+		taxonBase.getName().addSource(OriginalSourceType.Import, null,"TaxonName" ,state.getConfig().getSourceReference(), null);
+		taxonBase.addSource(OriginalSourceType.Import, null,"TaxonName" ,state.getConfig().getSourceReference(), null);
 
 		return taxonBase;
 	}
@@ -852,14 +853,16 @@ public class NormalExplicitImport extends TaxonExcelImporterBase {
 	//TODO implementation must be improved when matching of taxon names with existing names is implemented
 	//=> the assumption that the only description is the description added by this import
 	//is wrong then
-	private TaxonNameDescription getNameDescription(TaxonName name) {
+	private TaxonNameDescription getNameDescription(TaxonName name, TaxonExcelImportState state) {
 		Set<TaxonNameDescription> descriptions = name.getDescriptions();
 		if (descriptions.size()>1){
 			throw new IllegalStateException("Implementation does not yet support names with multiple descriptions");
 		}else if (descriptions.size()==1){
 			return descriptions.iterator().next();
 		}else{
-			return TaxonNameDescription.NewInstance(name);
+		    TaxonNameDescription desc = TaxonNameDescription.NewInstance(name);
+		    desc.addSource(OriginalSourceType.Import, null, "NameDescription", state.getConfig().getSourceReference(), null);
+			return desc;
 		}
 	}
 
diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/TaxonExcelImportState.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/TaxonExcelImportState.java
index 8d13b78b50..c4da42f997 100644
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/TaxonExcelImportState.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/TaxonExcelImportState.java
@@ -16,8 +16,8 @@ import java.util.Set;
 
 import org.apache.log4j.Logger;
 
-import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
 import eu.etaxonomy.cdm.io.excel.common.ExcelImportState;
+import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
 import eu.etaxonomy.cdm.model.taxon.Classification;
 import eu.etaxonomy.cdm.model.taxon.Taxon;
diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java
index db92c0f27f..ef782c0cb2 100755
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java
@@ -553,7 +553,9 @@ public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STA
 	        if (cdmBase.isInstanceOf(LanguageString.class)){
 	            return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 	        }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
-	            return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
+	            SpecimenOrObservationBase specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
+
+	            return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
 	        }else if (cdmBase.isInstanceOf(Reference.class)){
 	            return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 	        }else if (cdmBase.isInstanceOf(Classification.class)){
@@ -563,7 +565,9 @@ public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STA
 	        }else if (cdmBase.isInstanceOf(Collection.class)){
 	            return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 	        }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
-	            return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
+	            DescriptionBase description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
+
+	            return cdmRepository.getDescriptionService().saveOrUpdate(description);
 	        }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 	            return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 	        }else if (cdmBase.isInstanceOf(TaxonName.class)){
@@ -1008,14 +1012,14 @@ public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STA
 	        if (!existsInClassification(taxon,state.getClassification(), state)){
 	            if(config.isMoveNewTaxaToDefaultClassification()){
 	                //check if node exists in default classification
-	                if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
-	                    addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
+	                if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
+	                    addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
 	                }
-	            }
-	            else {
+	            }else{
 	                //add non-existing taxon to current classification
 	                addParentTaxon(taxon, state, preferredFlag, state.getClassification());
 	            }
+
 	        }
 	    }
 
diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportStateBase.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportStateBase.java
index e1bcd519a3..5145dd1eaf 100644
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportStateBase.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportStateBase.java
@@ -189,8 +189,8 @@ public class SpecimenImportStateBase<CONFIG extends SpecimenImportConfiguratorBa
         this.classification = classification;
     }
 
-    public Classification getDefaultClassification() {
-        if(defaultClassification==null){
+    public Classification getDefaultClassification(boolean createIfNotExist) {
+        if(defaultClassification==null && createIfNotExist){
             final String defaultClassificationAbcd = "Default Classification Specimen Import";
             for (Classification classif : cdmRepository.getClassificationService().list(Classification.class, null, null, null, null)){
                 if (classif.getTitleCache()!=null && classif.getTitleCache().equalsIgnoreCase(defaultClassificationAbcd)