From ae765538c26019388c4fe054e0c756e172ea56d8 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Andreas=20M=C3=BCller?= Date: Wed, 23 Mar 2022 14:51:33 +0100 Subject: [PATCH 1/1] ref #9932 latest changes to MexicoEflora imports --- .../cdm/app/mexico/MexicoEfloraActivator.java | 13 +- .../io/mexico/MexicoConabioTransformer.java | 1 + .../mexico/MexicoEfloraCommonNameImport.java | 30 ++--- .../MexicoEfloraCommonNameRefImport.java | 45 +++++-- .../MexicoEfloraDistributionImport.java | 13 +- .../MexicoEfloraDistributionRefImport.java | 48 +++++-- .../MexicoEfloraFactCategoryImport.java | 61 ++++----- .../cdm/io/mexico/MexicoEfloraFactImport.java | 46 ++++--- .../cdm/io/mexico/MexicoEfloraImportBase.java | 12 +- .../MexicoEfloraImportConfigurator.java | 21 +++- .../io/mexico/MexicoEfloraImportState.java | 18 +-- .../mexico/MexicoEfloraRefArticlesImport.java | 29 +---- .../MexicoEfloraRefOtherBooksImport.java | 38 ++---- .../mexico/MexicoEfloraRefSerialsImport.java | 36 ++---- .../mexico/MexicoEfloraRefWebSitesImport.java | 28 ----- .../MexicoEfloraReferenceImportBase.java | 61 +++++++-- .../io/mexico/MexicoEfloraRegionImport.java | 1 - .../io/mexico/MexicoEfloraTaxonImport.java | 119 ++++++++++-------- .../MexicoEfloraTaxonRelationImport.java | 11 +- 19 files changed, 320 insertions(+), 311 deletions(-) diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/app/mexico/MexicoEfloraActivator.java b/app-import/src/main/java/eu/etaxonomy/cdm/app/mexico/MexicoEfloraActivator.java index 4c8db46a..056a8517 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/app/mexico/MexicoEfloraActivator.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/app/mexico/MexicoEfloraActivator.java @@ -38,27 +38,27 @@ public class MexicoEfloraActivator { static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE; static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_cdmtest_mysql(); +// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_test_mysql(); // static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_test1(); // static final ICdmDataSource cdmDestination = CdmDestinations.cdm_mexico_flora(); + static final boolean doReferences = true; //feature tree uuid private static final UUID featureTreeUuid = UUID.fromString("dd2070ef-e75b-494b-bc6a-4a370271866b"); private static final UUID flatFeatureTreeUuid = UUID.fromString("33896d1f-c3df-4cdb-afb8-bffd35f9d6b0"); + private static final UUID flatFeatureTreeWithoutUsesUuid = UUID.fromString("61832cbd-3800-4d43-8048-9514cc7b7f26"); private static final String featureTreeName = "Catalogo feature tree"; private static final String flatFeatureTreeName = "Catalogo flat feature tree"; + private static final String flatFeatureTreeWithoutUsesName = "Catalogo flat feature tree without uses"; //classification static final UUID classificationUuid = UUID.fromString("588dd2de-7c95-42ad-8eb1-994460561b5e"); - private static final String classificationName = "Plantae"; + private static final String classificationName = "Plantas"; //check - import static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK; - boolean doVocabularies = (hbm2dll == DbSchemaValidation.CREATE); -// static final boolean doTaxa = false; -// static final boolean doDeduplicate = true; - protected void doImport(ICdmDataSource cdmDestination){ Source source = CdmImportSources.MEXICO_EFLORA(); @@ -72,10 +72,13 @@ public class MexicoEfloraActivator { config.setSecReference(getSecReference()); config.setFeatureTreeUuid(featureTreeUuid); config.setFlatFeatureTreeUuid(flatFeatureTreeUuid); + config.setFlatFeatureTreeWithoutUsesUuid(flatFeatureTreeWithoutUsesUuid); config.setFeatureTreeTitle(featureTreeName); config.setFlatFeatureTreeTitle(flatFeatureTreeName); + config.setFlatFeatureTreeWithoutUsesTitle(flatFeatureTreeWithoutUsesName); config.setRecordsPerTransaction(5000); + config.setDoReferences(doReferences); config.setSource(source); String fileName = source.toString(); diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoConabioTransformer.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoConabioTransformer.java index a37ed904..1568d6ff 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoConabioTransformer.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoConabioTransformer.java @@ -37,6 +37,7 @@ public class MexicoConabioTransformer extends InputTransformerBase{ public static final UUID uuidConabioTaxonIdIdentifierType = UUID.fromString("025fb8fe-e8e8-4136-bf37-93367a268889"); public static final UUID uuidConabioReferenceIdIdentifierType = UUID.fromString("a6f98411-b99b-45c5-9303-d2af421b9f73"); + public static final UUID uuidConabioFactCategoryIdIdentifierType = UUID.fromString("8d6c1c38-2d23-40b2-8891-f74501d3763c"); public static final UUID uuidNomRefExtension = UUID.fromString("0a7e2f5f-c62d-43e1-874a-07cb1dbb9fa0"); diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraCommonNameImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraCommonNameImport.java index 879b281c..09f1e609 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraCommonNameImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraCommonNameImport.java @@ -86,7 +86,7 @@ public class MexicoEfloraCommonNameImport extends MexicoEfloraImportBase { ResultSet rs = state.getConfig().getSource().getResultSet(sql); try { while (rs.next()) { - String lang = rs.getString("Lengua"); + String lang = rs.getString("Lengua").trim(); Language language = Language.NewInstance(lang, lang, null, Language.SPANISH_CASTILIAN()); voc.addTerm(language); getTermService().save(language); //not sure if necessary @@ -97,7 +97,7 @@ public class MexicoEfloraCommonNameImport extends MexicoEfloraImportBase { } } - private TermVocabulary createLanguagesVoc(MexicoEfloraImportState state) { + private TermVocabulary createLanguagesVoc(@SuppressWarnings("unused") MexicoEfloraImportState state) { URI termSourceUri = null; String label = "Mexican Languages"; String description = "Mexican languages as used by the CONABIO database"; @@ -130,12 +130,12 @@ public class MexicoEfloraCommonNameImport extends MexicoEfloraImportBase { //create TaxonName element String idCombi = rs.getString("IdCombinado"); - //TODO common name id -// String idNomComun = rs.getString("IdNomComun"); + + String idNomComun = rs.getString("IdNomComun"); String taxonUuid = rs.getString("taxonUuid"); String nomComunStr = rs.getString("NomComun"); - String lenguaStr = rs.getString("Lengua"); + String lenguaStr = rs.getString("Lengua").trim(); String paisStr = rs.getString("Pais"); String estadoStr = rs.getString("Estado"); @@ -155,12 +155,13 @@ public class MexicoEfloraCommonNameImport extends MexicoEfloraImportBase { NamedArea area = getArea(state, idRegion, estadoStr, paisStr, null, null); CommonTaxonName commonName = CommonTaxonName.NewInstance(nomComunStr, language, area); - //TODO source reference correct (everywhere?) + TaxonDescription description = this.getTaxonDescription(taxon, sourceReference, false, true); description.addElement(commonName); - state.getCommonNameMap().put(idCombi, commonName.getUuid()); + commonName.addImportSource(idNomComun + ";" + idCombi, "Eflora_NombresComunes4CDM", sourceReference, null); + state.getCommonNameMap().put(idCombi, commonName.getUuid()); partitioner.startDoSave(); taxaToSave.add(taxonBase); @@ -180,12 +181,10 @@ public class MexicoEfloraCommonNameImport extends MexicoEfloraImportBase { return success; } - - - private Language getLanguage(MexicoEfloraImportState state, String lenguaStr) { + private Language getLanguage(@SuppressWarnings("unused") MexicoEfloraImportState state, String lenguaStr) { Language language = languageMap.get(lenguaStr); if (isNotBlank(lenguaStr) && language == null) { - logger.warn("Language not found: " + lenguaStr); + logger.warn("Language not found: '" + lenguaStr + "'"); } return language; } @@ -194,15 +193,12 @@ public class MexicoEfloraCommonNameImport extends MexicoEfloraImportBase { public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { String nameSpace; - Set idSet; Map> result = new HashMap<>(); try{ Set taxonIdSet = new HashSet<>(); - Set referenceIdSet = new HashSet<>(); while (rs.next()){ handleForeignUuidKey(rs, taxonIdSet, "taxonUuid"); -// handleForeignKey(rs, referenceIdSet, "PTRefFk"); } //taxon map @@ -214,12 +210,6 @@ public class MexicoEfloraCommonNameImport extends MexicoEfloraImportBase { taxa.stream().forEach(t->taxonMap.put(t.getUuid().toString(), t)); result.put(nameSpace, taxonMap); - //reference map - nameSpace = MexicoEfloraRefArticlesImport.NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); - result.put(nameSpace, referenceMap); - } catch (SQLException e) { throw new RuntimeException(e); } diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraCommonNameRefImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraCommonNameRefImport.java index 1d12f046..1b7723e1 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraCommonNameRefImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraCommonNameRefImport.java @@ -24,6 +24,7 @@ import eu.etaxonomy.cdm.io.common.ResultSetPartitioner; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.description.CommonTaxonName; import eu.etaxonomy.cdm.model.description.DescriptionElementSource; +import eu.etaxonomy.cdm.model.description.TaxonDescription; import eu.etaxonomy.cdm.model.name.TaxonName; import eu.etaxonomy.cdm.model.reference.Reference; @@ -92,15 +93,13 @@ public class MexicoEfloraCommonNameRefImport extends MexicoEfloraImportBase { CommonTaxonName commonName = commonNameMap.get(idCombi); Reference ref = referenceMap.get(String.valueOf(idBibliografia)); -// String detail = state.getRefDetailMap().get(idBibliografia); String detail = null; if (commonName != null) { DescriptionElementSource source = commonName.addPrimaryTaxonomicSource(ref, detail); if (source!= null) { - TaxonName nameUsedInSource = getNameUsedInSource(state, observaciones); + TaxonName nameUsedInSource = getNameUsedInSource(state, observaciones, commonName, ref); source.setNameUsedInSource(nameUsedInSource); - //TODO other observaciones } else { logger.warn("Source not found for " + idCombi + " and bibID: " + idBibliografia); } @@ -125,16 +124,33 @@ public class MexicoEfloraCommonNameRefImport extends MexicoEfloraImportBase { return success; } - private TaxonName getNameUsedInSource(MexicoEfloraImportState state, String observaciones) { - // TODO named used in source for common names + private TaxonName getNameUsedInSource(MexicoEfloraImportState state, String observaciones, CommonTaxonName commonName, Reference ref) { + if (observaciones != null) { + if (observaciones.matches("^(reportada |\\()?como .*")){ + String nameStr = observaciones + .replaceAll("^(reportada |\\()?como ", "") + .replaceAll("\\)$", ""); + UUID nameUuid = state.getNameMap().get(nameStr); + TaxonName name = getName(state, nameUuid); + if (name == null) { + String taxon = CdmBase.deproxy(commonName.getInDescription(),TaxonDescription.class).getTaxon().getName().getTitleCache(); + logger.warn("Name in source ("+observaciones+") could not be found for " + taxon + "-" + commonName.getLanguage().getLabel() + "-" + commonName.getArea().getLabel() + "-" + ref.getTitleCache()); + } + return name; + } + } return null; } + //quick and dirty and slow + private TaxonName getName(@SuppressWarnings("unused") MexicoEfloraImportState state, UUID nameUuid) { + return getNameService().find(nameUuid); + } + @Override public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { String nameSpace; - Set idSet; Map> result = new HashMap<>(); try{ @@ -147,18 +163,21 @@ public class MexicoEfloraCommonNameRefImport extends MexicoEfloraImportBase { //common name map nameSpace = MexicoEfloraCommonNameImport.NAMESPACE; - Map uuidMap = new HashMap<>(); - commonNameIdSet.stream().forEach(cnId->uuidMap.put(state.getCommonNameMap().get(cnId),cnId)); + Map commonNameUuidMap = new HashMap<>(); + commonNameIdSet.stream().forEach(cnId->commonNameUuidMap.put(state.getCommonNameMap().get(cnId), cnId)); @SuppressWarnings({ "rawtypes", "unchecked" }) - List commonNames = (List)getDescriptionElementService().find(uuidMap.keySet()); + List commonNames = (List)getDescriptionElementService().find(commonNameUuidMap.keySet()); Map commonNameMap = new HashMap<>(); - commonNames.stream().forEach(cn->commonNameMap.put(uuidMap.get(cn.getUuid()), cn)); + commonNames.stream().forEach(cn->commonNameMap.put(commonNameUuidMap.get(cn.getUuid()), cn)); result.put(nameSpace, commonNameMap); - //reference map + //reference map nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); + Map referenceUuidMap = new HashMap<>(); + referenceIdSet.stream().forEach(rId->referenceUuidMap.put(state.getReferenceUuidMap().get(Integer.valueOf(rId)), rId)); + List references = getReferenceService().find(referenceUuidMap.keySet()); + Map referenceMap = new HashMap<>(); + references.stream().forEach(r->referenceMap.put(referenceUuidMap.get(r.getUuid()), r)); result.put(nameSpace, referenceMap); } catch (SQLException e) { diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraDistributionImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraDistributionImport.java index 486d39d3..005ba4a6 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraDistributionImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraDistributionImport.java @@ -90,9 +90,9 @@ public class MexicoEfloraDistributionImport extends MexicoEfloraImportBase { //create TaxonName element String idCombi = rs.getString("IdDist"); + String taxonUuid = rs.getString("taxonUuid"); - //TODO nombre in distribution gegenchecken -// String nombreStr = rs.getString("Nombre"); + String nombreStr = rs.getString("Nombre"); String paisStr = rs.getString("Pais"); String estadoStr = rs.getString("Estado"); String abreviaturaEstado = rs.getString("AbreviaturaEstado"); @@ -102,10 +102,13 @@ public class MexicoEfloraDistributionImport extends MexicoEfloraImportBase { // int idTipoDistribucion = rs.getInt("IdTipoDistribucion"); int idTipoRegion = rs.getInt("IdTipoRegion"); - //FIXME mapping to mapservice - try { TaxonBase taxonBase = taxonMap.get(taxonUuid); + if(isNotBlank(nombreStr) && taxonBase != null && taxonBase.getName() != null + && !nombreStr.equals(taxonBase.getName().getNameCache()) + && !nombreStr.contains("(") && !taxonBase.getName().isHybrid()) { + logger.warn(idCombi + ": Name differs " + nombreStr + "<->" + taxonBase.getName().getNameCache()); + } Taxon taxon; if (taxonBase == null) { logger.warn("Taxon "+taxonUuid+" not found for distribution " + idCombi); @@ -122,11 +125,11 @@ public class MexicoEfloraDistributionImport extends MexicoEfloraImportBase { Distribution distribution = Distribution.NewInstance(area, status); - //TODO source reference correct (everywhere?) TaxonDescription description = this.getTaxonDescription( taxon, sourceReference, false, true); description.addElement(distribution); + distribution.addImportSource(idCombi, "Eflora_DistribucionEstatalIndividual", sourceReference, null); state.getDistributionMap().put(idCombi, distribution.getUuid()); partitioner.startDoSave(); diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraDistributionRefImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraDistributionRefImport.java index 026077c2..9951f2ac 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraDistributionRefImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraDistributionRefImport.java @@ -24,6 +24,7 @@ import eu.etaxonomy.cdm.io.common.ResultSetPartitioner; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.description.DescriptionElementSource; import eu.etaxonomy.cdm.model.description.Distribution; +import eu.etaxonomy.cdm.model.description.TaxonDescription; import eu.etaxonomy.cdm.model.name.TaxonName; import eu.etaxonomy.cdm.model.reference.Reference; @@ -93,15 +94,12 @@ public class MexicoEfloraDistributionRefImport extends MexicoEfloraImportBase { continue; } Reference ref = referenceMap.get(String.valueOf(idBibliografia)); -// String detail = state.getRefDetailMap().get(idBibliografia); String detail = null; DescriptionElementSource source = distribution.addPrimaryTaxonomicSource(ref, detail); - //TODO nameUsedInSource for distribution if (source!= null) { - TaxonName nameUsedInSource = getNameUsedInSource(state, observaciones); + TaxonName nameUsedInSource = getNameUsedInSource(state, observaciones, distribution, ref); source.setNameUsedInSource(nameUsedInSource); - //TODO other observaciones } else { logger.warn("Source not found for " + idCombi + " and bibID: " + idBibliografia); } @@ -123,16 +121,37 @@ public class MexicoEfloraDistributionRefImport extends MexicoEfloraImportBase { return success; } - private TaxonName getNameUsedInSource(MexicoEfloraImportState state, String observaciones) { - // TODO Auto-generated method stub + private TaxonName getNameUsedInSource(MexicoEfloraImportState state, String observaciones, Distribution distribution, Reference ref) { + if (observaciones != null) { + if (observaciones.matches("^\\(?como .*")){ + String nameStr = observaciones + .replaceAll("^\\(?como ", "") + .replaceAll("\\)$", "").trim(); + if (nameStr.contains("registro obtenido a partir")) { + nameStr = nameStr.substring(0, nameStr.indexOf("registro obtenido a partir")).trim(); + } + + UUID nameUuid = state.getNameMap().get(nameStr); + TaxonName name = getName(state, nameUuid); + if (name == null) { + String taxon = CdmBase.deproxy(distribution.getInDescription(),TaxonDescription.class).getTaxon().getName().getTitleCache(); + logger.warn("Name in source ("+observaciones+") could not be found for " + taxon + " - Area: " + distribution.getArea().getLabel() + " - Biblio: " + ref.getTitleCache()); + } + return name; + } + } return null; } + //quick and dirty and slow + private TaxonName getName(@SuppressWarnings("unused") MexicoEfloraImportState state, UUID nameUuid) { + return getNameService().find(nameUuid); + } + @Override public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { String nameSpace; - Set idSet; Map> result = new HashMap<>(); try{ @@ -145,18 +164,21 @@ public class MexicoEfloraDistributionRefImport extends MexicoEfloraImportBase { //distribution map nameSpace = MexicoEfloraDistributionImport.NAMESPACE; - Map uuidMap = new HashMap<>(); - distributionIdSet.stream().forEach(dId->uuidMap.put(state.getDistributionMap().get(dId),dId)); + Map distributionUuidMap = new HashMap<>(); + distributionIdSet.stream().forEach(dId->distributionUuidMap.put(state.getDistributionMap().get(dId),dId)); @SuppressWarnings({ "rawtypes", "unchecked" }) - List distributions = (List)getDescriptionElementService().find(uuidMap.keySet()); + List distributions = (List)getDescriptionElementService().find(distributionUuidMap.keySet()); Map distributionMap = new HashMap<>(); - distributions.stream().forEach(d->distributionMap.put(uuidMap.get(d.getUuid()), d)); + distributions.stream().forEach(d->distributionMap.put(distributionUuidMap.get(d.getUuid()), d)); result.put(nameSpace, distributionMap); //reference map nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); + Map referenceUuidMap = new HashMap<>(); + referenceIdSet.stream().forEach(rId->referenceUuidMap.put(state.getReferenceUuidMap().get(Integer.valueOf(rId)), rId)); + List references = getReferenceService().find(referenceUuidMap.keySet()); + Map referenceMap = new HashMap<>(); + references.stream().forEach(r->referenceMap.put(referenceUuidMap.get(r.getUuid()), r)); result.put(nameSpace, referenceMap); } catch (SQLException e) { diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraFactCategoryImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraFactCategoryImport.java index c5705b60..b5637f93 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraFactCategoryImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraFactCategoryImport.java @@ -12,25 +12,19 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import org.springframework.transaction.TransactionStatus; import eu.etaxonomy.cdm.common.UTF8; -import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport; -import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonNameImport; import eu.etaxonomy.cdm.io.common.ResultSetPartitioner; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.common.Language; import eu.etaxonomy.cdm.model.description.Feature; import eu.etaxonomy.cdm.model.description.State; -import eu.etaxonomy.cdm.model.name.TaxonName; -import eu.etaxonomy.cdm.model.reference.Reference; import eu.etaxonomy.cdm.model.term.TermNode; import eu.etaxonomy.cdm.model.term.TermTree; import eu.etaxonomy.cdm.model.term.TermType; @@ -112,21 +106,26 @@ public class MexicoEfloraFactCategoryImport extends MexicoEfloraImportBase { //hierarchical feature tree TermTree featureTree = TermTree.NewFeatureInstance(state.getConfig().getFeatureTreeUuid()); featureTree.setLabel(state.getConfig().getFeatureTreeTitle(), Language.SPANISH_CASTILIAN()); - featureTree.setUuid(state.getConfig().getFeatureTreeUuid()); featureTree.getRoot().addChild(Feature.DISTRIBUTION()); getTermTreeService().save(featureTree); //flat feature tree - TermTree flatFeatureTree = TermTree.NewFeatureInstance(state.getConfig().getFeatureTreeUuid()); + TermTree flatFeatureTree = TermTree.NewFeatureInstance(state.getConfig().getFlatFeatureTreeUuid()); flatFeatureTree.setLabel(state.getConfig().getFlatFeatureTreeTitle(), Language.SPANISH_CASTILIAN()); - flatFeatureTree.setUuid(state.getConfig().getFlatFeatureTreeUuid()); flatFeatureTree.getRoot().addChild(Feature.DISTRIBUTION()); getTermTreeService().save(flatFeatureTree); + //flat feature tree without Usos + TermTree flatFeatureTreeWithoutUses = TermTree.NewFeatureInstance(state.getConfig().getFlatFeatureTreeWithoutUsesUuid()); + flatFeatureTreeWithoutUses.setLabel(state.getConfig().getFlatFeatureTreeWithoutUsesTitle(), Language.SPANISH_CASTILIAN()); + flatFeatureTreeWithoutUses.getRoot().addChild(Feature.DISTRIBUTION()); + getTermTreeService().save(flatFeatureTreeWithoutUses); getVocabularyService().save(featureVoc); for (TreeNode child : root.children) { - saveNodeRecursive(state, child, featureVoc, null, featureTree.getRoot(), flatFeatureTree.getRoot()); + //filter out Uses + TermNode withoutUsesRoot = child.idCatNombre == 460? null : flatFeatureTreeWithoutUses.getRoot(); + saveNodeRecursive(state, child, featureVoc, null, featureTree.getRoot(), flatFeatureTree.getRoot(), withoutUsesRoot); } featureTree.getRoot().addChild(Feature.COMMON_NAME()); @@ -135,8 +134,10 @@ public class MexicoEfloraFactCategoryImport extends MexicoEfloraImportBase { this.commitTransaction(tx); } - private void saveNodeRecursive(MexicoEfloraImportState state, TreeNode node, TermVocabulary featureVoc, Feature parentFeature, - TermNode parentFeatureTreeNode, TermNode flatFeatureTreereTreeNode) { + private void saveNodeRecursive(MexicoEfloraImportState state, TreeNode node, + TermVocabulary featureVoc, Feature parentFeature, + TermNode parentFeatureTreeNode, TermNode flatFeatureTreeTreeNode, + TermNode flatFeatureTreeWithoutUsesTreeNode) { if (!node.children.isEmpty()) { //is feature @@ -145,11 +146,15 @@ public class MexicoEfloraFactCategoryImport extends MexicoEfloraImportBase { label = label.startsWith(sep)? label.substring(sep.length()):label; Feature feature = Feature.NewInstance(label, label, String.valueOf(node.idCatNombre), Language.SPANISH_CASTILIAN()); feature.setIdInVocabulary(node.key); + feature.addIdentifier(String.valueOf(node.idCatNombre), getIdentiferType(state, MexicoConabioTransformer.uuidConabioFactCategoryIdIdentifierType, "CONABIO Fact Identifier", "CONABIO Fact Identifier", null, null)); feature.setSupportsCategoricalData(true); feature.setSupportsTextData(false); featureVoc.addTerm(feature); TermNode featureTreeNode = parentFeatureTreeNode.addChild(feature); - flatFeatureTreereTreeNode.addChild(feature); + flatFeatureTreeTreeNode.addChild(feature); + if (flatFeatureTreeWithoutUsesTreeNode != null) { + flatFeatureTreeWithoutUsesTreeNode.addChild(feature); + } getTermService().save(feature); //parent-child if (parentFeature != null) { @@ -157,7 +162,7 @@ public class MexicoEfloraFactCategoryImport extends MexicoEfloraImportBase { parentFeature.addIncludes(feature); } for (TreeNode child : node.children) { - saveNodeRecursive(state, child, featureVoc, feature, featureTreeNode, flatFeatureTreereTreeNode); + saveNodeRecursive(state, child, featureVoc, feature, featureTreeNode, flatFeatureTreeTreeNode, flatFeatureTreeWithoutUsesTreeNode); } } // else { @@ -173,6 +178,8 @@ public class MexicoEfloraFactCategoryImport extends MexicoEfloraImportBase { } State categoricalState = State.NewInstance(node.description, node.description, String.valueOf(node.idCatNombre), Language.SPANISH_CASTILIAN()); categoricalState.setIdInVocabulary(node.key); + categoricalState.addIdentifier(String.valueOf(node.idCatNombre), getIdentiferType(state, MexicoConabioTransformer.uuidConabioFactCategoryIdIdentifierType, "CONABIO Fact Identifier", "CONABIO Fact Identifier", null, null)); + state.getStateMap().put(node.idCatNombre, categoricalState); state.getFeatureMap().put(node.idCatNombre, parentFeature); getTermService().save(categoricalState); @@ -268,33 +275,7 @@ public class MexicoEfloraFactCategoryImport extends MexicoEfloraImportBase { @Override public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { - String nameSpace; - Set idSet; Map> result = new HashMap<>(); - - try{ - Set nameIdSet = new HashSet<>(); - Set referenceIdSet = new HashSet<>(); - while (rs.next()){ -// handleForeignKey(rs, nameIdSet, "PTNameFk"); -// handleForeignKey(rs, referenceIdSet, "PTRefFk"); - } - - //name map - nameSpace = BerlinModelTaxonNameImport.NAMESPACE; - idSet = nameIdSet; - Map nameMap = getCommonService().getSourcedObjectsByIdInSourceC(TaxonName.class, idSet, nameSpace); - result.put(nameSpace, nameMap); - - //reference map - nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); - result.put(nameSpace, referenceMap); - - } catch (SQLException e) { - throw new RuntimeException(e); - } return result; } diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraFactImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraFactImport.java index e22b9765..7eadb4cb 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraFactImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraFactImport.java @@ -51,9 +51,10 @@ public class MexicoEfloraFactImport extends MexicoEfloraImportBase { @Override protected String getIdQuery(MexicoEfloraImportState state) { - String sql = " SELECT id " - + " FROM " + dbTableName - + " ORDER BY IdCAT, IdCatNombre, IdBibliografia "; + String sql = " SELECT f.id " + + " FROM " + dbTableName + " f " + + " LEFT JOIN Eflora_CatalogoNombre4CDM c ON c.IdCatNombre = f.IdCatNombre" + + " ORDER BY f.IdCAT, c.Nivel1, c.Nivel2, c.Nivel3, c.Nivel4, c.Nivel5, c.Nivel6, c.Nivel7, f.IdCatNombre, f.IdBibliografia "; return sql; } @@ -71,6 +72,9 @@ public class MexicoEfloraFactImport extends MexicoEfloraImportBase { private CategoricalData lastFact; private String lastIdCat = "-1"; private int lastIdCatNombre = -1; + @SuppressWarnings("unused") + private int lastBiblioId = -1; + @Override public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) { @@ -99,14 +103,16 @@ public class MexicoEfloraFactImport extends MexicoEfloraImportBase { String uuidTaxonStr = rs.getString("taxonUuid"); int idBibliografia = rs.getInt("IdBibliografia"); - //TODO observaciones in facts - String observaciones = rs.getString("Observaciones"); + //handled in Excel log +// String observaciones = rs.getString("Observaciones"); try { CategoricalData categoricalData; if (idCAT.equals(lastIdCat) && idCatNombre == lastIdCatNombre) { + //same taxon, same feature, same state => only add bibliography categoricalData = lastFact; }else { + //create to potential categorical data categoricalData = makeCategoricalData(state, idCatNombre); Feature lastFeature = lastFact == null? null : lastFact.getFeature(); if (idCAT.equals(lastIdCat) && categoricalData.getFeature().equals(lastFeature)) { @@ -114,11 +120,11 @@ public class MexicoEfloraFactImport extends MexicoEfloraImportBase { //add the single new state to the existing categorical data //TODO not fully correct if bibliography differs for the single states; State newState = categoricalData.getStatesOnly().stream().findFirst().orElse(null); - if (newState != null) { + if (newState != null && !lastFact.hasState(newState)) { lastFact.addStateData(newState); } categoricalData = lastFact; -// lastIdCatNombre = idCatNombre; + lastIdCatNombre = idCatNombre; }else { //new categorical data TaxonBase taxonBase = taxonMap.get(uuidTaxonStr); @@ -130,16 +136,18 @@ public class MexicoEfloraFactImport extends MexicoEfloraImportBase { continue; } - //TODO source reference correct? TaxonDescription description = this.getTaxonDescription(taxon, sourceReference, false, true); + description.addElement(categoricalData); - lastFact = categoricalData; - lastIdCat = idCAT; - lastIdCatNombre = idCatNombre; + categoricalData.addImportSource(idCAT+";"+idCatNombre, "Eflora_RelBiblioNombreCatalogoNombre", sourceReference, null); } } handleBibliografia(state, referenceMap, categoricalData, idBibliografia, id); + lastBiblioId = idBibliografia; + lastIdCat = idCAT; + lastIdCatNombre = idCatNombre; + lastFact = categoricalData; partitioner.startDoSave(); } catch (Exception e) { @@ -157,7 +165,7 @@ public class MexicoEfloraFactImport extends MexicoEfloraImportBase { return success; } - private void handleBibliografia(MexicoEfloraImportState state, Map referenceMap, + private void handleBibliografia(@SuppressWarnings("unused") MexicoEfloraImportState state, Map referenceMap, CategoricalData categoricalData, int idBibliografia, int id) { Reference ref = referenceMap == null ? null : referenceMap.get(String.valueOf(idBibliografia)); @@ -199,7 +207,6 @@ public class MexicoEfloraFactImport extends MexicoEfloraImportBase { public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { String nameSpace; - Set idSet; Map> result = new HashMap<>(); try{ @@ -219,11 +226,14 @@ public class MexicoEfloraFactImport extends MexicoEfloraImportBase { taxa.stream().forEach(t->taxonMap.put(t.getUuid().toString(), t)); result.put(nameSpace, taxonMap); - //reference map - nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); - result.put(nameSpace, referenceMap); + //reference map + nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE; + Map referenceUuidMap = new HashMap<>(); + referenceIdSet.stream().forEach(rId->referenceUuidMap.put(state.getReferenceUuidMap().get(Integer.valueOf(rId)), rId)); + List references = getReferenceService().find(referenceUuidMap.keySet()); + Map referenceMap = new HashMap<>(); + references.stream().forEach(r->referenceMap.put(referenceUuidMap.get(r.getUuid()), r)); + result.put(nameSpace, referenceMap); } catch (SQLException e) { throw new RuntimeException(e); diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportBase.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportBase.java index 2a3b8267..526e5770 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportBase.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportBase.java @@ -59,7 +59,7 @@ public abstract class MexicoEfloraImportBase String estadoStr, String paisStr, String abbrev, Integer idTipoRegion) { NamedArea areaById = state.getAreaMap().get(idRegion); - if (areaIsMexicoCountry(idRegion, estadoStr, paisStr, idTipoRegion)) { + if (areaIsMexicoCountry(idRegion, estadoStr, paisStr)) { return areaById; } String label = isNotBlank(estadoStr) ? estadoStr : paisStr; @@ -80,14 +80,13 @@ public abstract class MexicoEfloraImportBase logger.warn("Area not found, create new one: " + idRegion + "; " + label); NamedAreaLevel level = idTipoRegion != null && idTipoRegion.equals(1)? NamedAreaLevel.COUNTRY() : null; NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA(); - //TODO new namedAreas vocabulary NamedArea namedArea = this.getNamedArea(state, null, label, label, abbrev, areaType, level, null, null); state.getAreaLabelMap().put(labelKey, namedArea); } return result; } - private NamedArea getNewCountry(MexicoEfloraImportState state, String label) { + private NamedArea getNewCountry(@SuppressWarnings("unused") MexicoEfloraImportState state, String label) { if (StringUtils.isBlank(label)) { return null; }else if (label.equalsIgnoreCase("CANADÁ")) {return Country.CANADA(); @@ -134,13 +133,16 @@ public abstract class MexicoEfloraImportBase }else if (label.equalsIgnoreCase("ANGOLA")) {return Country.ANGOLAREPUBLICOF(); }else if (label.equalsIgnoreCase("TAILANDIA")) {return Country.THAILANDKINGDOMOF(); }else if (label.equalsIgnoreCase("TANZANIA")) {return Country.TANZANIAUNITEDREPUBLICOF(); + }else if (label.equalsIgnoreCase("ARGENTINA")) {return Country.ARGENTINAARGENTINEREPUBLIC(); + }else if (label.equalsIgnoreCase("SURINAM")) {return Country.SURINAMEREPUBLICOF(); + }else if (label.equalsIgnoreCase("INGLATERRA")) {return Country.UNITEDKINGDOMOFGREATBRITAINANDNORTHERNIRELAND(); } return null; } private boolean areaIsMexicoCountry(Integer idRegion, String estadoStr, - String paisStr, Integer idTipoRegion) { + String paisStr) { return idRegion == 2 && isBlank(estadoStr) && - "MÉXICO".equalsIgnoreCase(paisStr); // && idTipoRegion == 1; + "MÉXICO".equalsIgnoreCase(paisStr); } } \ No newline at end of file diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportConfigurator.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportConfigurator.java index aee6e466..0e97c647 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportConfigurator.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportConfigurator.java @@ -37,7 +37,6 @@ public class MexicoEfloraImportConfigurator return new MexicoEfloraImportConfigurator(berlinModelSource, destination); } - //TODO private static IInputTransformer defaultTransformer = new MexicoConabioTransformer(); private boolean doCommonNames = true; @@ -53,10 +52,11 @@ public class MexicoEfloraImportConfigurator private UUID featureTreeUuid; private UUID flatFeatureTreeUuid; + private UUID flatFeatureTreeWithoutUsesUuid; private String featureTreeTitle; private String flatFeatureTreeTitle; - + private String flatFeatureTreeWithoutUsesTitle; private Reference secReference; @@ -90,13 +90,11 @@ public class MexicoEfloraImportConfigurator super(berlinModelSource, destination, NomenclaturalCode.ICNAFP, defaultTransformer); //default for Berlin Model } - public boolean isDoCommonNames() { return doCommonNames; } public void setDoCommonNames(boolean doCommonNames) { this.doCommonNames = doCommonNames; - } public boolean isDoFacts() { @@ -156,6 +154,13 @@ public class MexicoEfloraImportConfigurator this.flatFeatureTreeUuid = flatFeatureTreeUuid; } + public UUID getFlatFeatureTreeWithoutUsesUuid() { + return flatFeatureTreeWithoutUsesUuid; + } + public void setFlatFeatureTreeWithoutUsesUuid(UUID flatFeatureTreeWithoutUsesUuid) { + this.flatFeatureTreeWithoutUsesUuid = flatFeatureTreeWithoutUsesUuid; + } + @Override public String getFeatureTreeTitle() { return featureTreeTitle; @@ -172,6 +177,13 @@ public class MexicoEfloraImportConfigurator this.flatFeatureTreeTitle = flatFeatureTreeTitle; } + public String getFlatFeatureTreeWithoutUsesTitle() { + return flatFeatureTreeWithoutUsesTitle; + } + public void setFlatFeatureTreeWithoutUsesTitle(String flatFeatureTreeWithoutUsesTitle) { + this.flatFeatureTreeWithoutUsesTitle = flatFeatureTreeWithoutUsesTitle; + } + public boolean isDoNamedAreas() { return doNamedAreas; } @@ -186,4 +198,5 @@ public class MexicoEfloraImportConfigurator this.secReference = secReference; } + } diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportState.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportState.java index a6d036de..95cf3232 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportState.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraImportState.java @@ -33,7 +33,7 @@ public class MexicoEfloraImportState @SuppressWarnings("unused") private static final Logger logger = Logger.getLogger(MexicoEfloraImportState.class); - private Map dbCdmDefTermMap = new HashMap<>(); + private Map> dbCdmDefTermMap = new HashMap<>(); private Map featureMap = new HashMap<>(); private Map stateMap = new HashMap<>(); @@ -41,25 +41,26 @@ public class MexicoEfloraImportState private Map areaLabelMap = new HashMap<>(); private Map referenceUuidMap = new HashMap<>(); -// private Map refDetailMap = new HashMap<>(); private Map commonNameMap = new HashMap<>(); private Map distributionMap = new HashMap<>(); + private Map nameMap = new HashMap<>(); + public MexicoEfloraImportState(MexicoEfloraImportConfigurator config) { super(config); } - public Map getDbCdmDefinedTermMap(){ + public Map> getDbCdmDefinedTermMap(){ return this.dbCdmDefTermMap; } - public void putDefinedTermToMap(String tableName, String id, DefinedTermBase term){ + public void putDefinedTermToMap(String tableName, String id, DefinedTermBase term){ this.dbCdmDefTermMap.put(tableName + "_" + id, term); } - public void putDefinedTermToMap(String tableName, int id, DefinedTermBase term){ + public void putDefinedTermToMap(String tableName, int id, DefinedTermBase term){ putDefinedTermToMap(tableName, String.valueOf(id), term); } @@ -81,9 +82,6 @@ public class MexicoEfloraImportState public Map getReferenceUuidMap() { return referenceUuidMap; } -// public Map getRefDetailMap() { -// return refDetailMap; -// } public Map getCommonNameMap() { return commonNameMap; @@ -93,5 +91,7 @@ public class MexicoEfloraImportState } - + public Map getNameMap() { + return nameMap; + } } diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefArticlesImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefArticlesImport.java index a09b786e..998b621d 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefArticlesImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefArticlesImport.java @@ -10,7 +10,6 @@ package eu.etaxonomy.cdm.io.mexico; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -19,7 +18,6 @@ import java.util.Set; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport; import eu.etaxonomy.cdm.io.common.ResultSetPartitioner; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.reference.IJournal; @@ -37,8 +35,6 @@ public class MexicoEfloraRefArticlesImport extends MexicoEfloraReferenceImportBa private static final Logger logger = Logger.getLogger(MexicoEfloraRefArticlesImport.class); - public static final String NAMESPACE = "Articles"; - private static final String pluralString = "Articles"; private static final String dbTableName = "RefArticles"; private static final Map journalMap = new HashMap<>(); @@ -51,12 +47,8 @@ public class MexicoEfloraRefArticlesImport extends MexicoEfloraReferenceImportBa public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) { boolean success = true ; - MexicoEfloraImportConfigurator config = state.getConfig(); Set refsToSave = new HashSet<>(); - @SuppressWarnings("unchecked") - Map refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE); - ResultSet rs = partitioner.getResultSet(); try{ while (rs.next()){ @@ -71,6 +63,7 @@ public class MexicoEfloraRefArticlesImport extends MexicoEfloraReferenceImportBa String articleTitleStr = rs.getString("ArticleTitle"); String journalTitleStr = rs.getString("JournalTitle"); String concat = rs.getString("Concatenation"); + //TODO _minor V and P in articles String vStr = rs.getString("V"); String pStr = rs.getString("P"); String urlStr = rs.getString("URL"); @@ -160,27 +153,7 @@ public class MexicoEfloraRefArticlesImport extends MexicoEfloraReferenceImportBa @Override public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { - String nameSpace; - Set idSet; Map> result = new HashMap<>(); - - try{ - Set nameIdSet = new HashSet<>(); - Set referenceIdSet = new HashSet<>(); - while (rs.next()){ -// handleForeignKey(rs, nameIdSet, "PTNameFk"); -// handleForeignKey(rs, referenceIdSet, "PTRefFk"); - } - - //reference map - nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); - result.put(nameSpace, referenceMap); - - } catch (SQLException e) { - throw new RuntimeException(e); - } return result; } diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefOtherBooksImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefOtherBooksImport.java index 06114d4f..06fb52d9 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefOtherBooksImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefOtherBooksImport.java @@ -10,7 +10,6 @@ package eu.etaxonomy.cdm.io.mexico; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -19,7 +18,6 @@ import java.util.Set; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport; import eu.etaxonomy.cdm.io.common.ResultSetPartitioner; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.reference.Reference; @@ -35,8 +33,6 @@ public class MexicoEfloraRefOtherBooksImport extends MexicoEfloraReferenceImport private static final long serialVersionUID = -1186364983750790695L; private static final Logger logger = Logger.getLogger(MexicoEfloraRefOtherBooksImport.class); - public static final String NAMESPACE = "OtherBooks"; - private static final String pluralString = "Other books"; private static final String dbTableName = "RefOtherBooks"; @@ -48,11 +44,7 @@ public class MexicoEfloraRefOtherBooksImport extends MexicoEfloraReferenceImport public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) { boolean success = true ; - MexicoEfloraImportConfigurator config = state.getConfig(); - Set refsToSave = new HashSet<>(); - - @SuppressWarnings("unchecked") - Map refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE); + Set refsToSave = new HashSet<>(); ResultSet rs = partitioner.getResultSet(); try{ @@ -68,7 +60,8 @@ public class MexicoEfloraRefOtherBooksImport extends MexicoEfloraReferenceImport String titleStr = rs.getString("Title"); String volumeStr = rs.getString("Volume"); String pagesStr = rs.getString("Pages"); - String observacionesStr = rs.getString("Observaciones"); + //exported as Excel table +// String observacionesStr = rs.getString("Observaciones"); String urlStr = rs.getString("URL"); String doiStr = rs.getString("DOI"); String isbnStr = rs.getString("ISBN"); @@ -89,6 +82,11 @@ public class MexicoEfloraRefOtherBooksImport extends MexicoEfloraReferenceImport //title handleTitleStr(state, titleStr, ref, refId); + //pages + if (isNotBlank(pagesStr)) { + ref.setPages(pagesStr); + } + //concat if (isNotBlank(volumeStr)) { ref.setVolume(volumeStr); @@ -134,27 +132,7 @@ public class MexicoEfloraRefOtherBooksImport extends MexicoEfloraReferenceImport @Override public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { - String nameSpace; - Set idSet; Map> result = new HashMap<>(); - - try{ - Set nameIdSet = new HashSet<>(); - Set referenceIdSet = new HashSet<>(); - while (rs.next()){ -// handleForeignKey(rs, nameIdSet, "PTNameFk"); -// handleForeignKey(rs, referenceIdSet, "PTRefFk"); - } - - //reference map - nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); - result.put(nameSpace, referenceMap); - - } catch (SQLException e) { - throw new RuntimeException(e); - } return result; } diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefSerialsImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefSerialsImport.java index 70aedb10..e23da018 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefSerialsImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefSerialsImport.java @@ -10,7 +10,6 @@ package eu.etaxonomy.cdm.io.mexico; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -19,7 +18,6 @@ import java.util.Set; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport; import eu.etaxonomy.cdm.io.common.ResultSetPartitioner; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.reference.Reference; @@ -49,11 +47,7 @@ public class MexicoEfloraRefSerialsImport extends MexicoEfloraReferenceImportBas public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) { boolean success = true ; - MexicoEfloraImportConfigurator config = state.getConfig(); - Set refsToSave = new HashSet<>(); - - @SuppressWarnings("unchecked") - Map refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE); + Set refsToSave = new HashSet<>(); ResultSet rs = partitioner.getResultSet(); try{ @@ -69,7 +63,8 @@ public class MexicoEfloraRefSerialsImport extends MexicoEfloraReferenceImportBas String titleStr = rs.getString("Title"); String volumeStr = rs.getString("Volume"); String pagesStr = rs.getString("Pages"); - String observacionesStr = rs.getString("Observaciones"); + //exported as Excel table +// String observacionesStr = rs.getString("Observaciones"); String urlStr = rs.getString("URL"); String doiStr = rs.getString("DOI"); String isbnStr = rs.getString("ISBN"); @@ -90,6 +85,11 @@ public class MexicoEfloraRefSerialsImport extends MexicoEfloraReferenceImportBas //title handleTitleStr(state, titleStr, ref, refId); + //pages + if (isNotBlank(pagesStr)) { + ref.setPages(pagesStr); + } + //concat if (isNotBlank(volumeStr)) { ref.setVolume(volumeStr); @@ -136,27 +136,7 @@ public class MexicoEfloraRefSerialsImport extends MexicoEfloraReferenceImportBas @Override public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { - String nameSpace; - Set idSet; Map> result = new HashMap<>(); - - try{ - Set nameIdSet = new HashSet<>(); - Set referenceIdSet = new HashSet<>(); - while (rs.next()){ -// handleForeignKey(rs, nameIdSet, "PTNameFk"); -// handleForeignKey(rs, referenceIdSet, "PTRefFk"); - } - - //reference map - nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); - result.put(nameSpace, referenceMap); - - } catch (SQLException e) { - throw new RuntimeException(e); - } return result; } diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefWebSitesImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefWebSitesImport.java index a7a2caf9..aa426679 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefWebSitesImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRefWebSitesImport.java @@ -9,7 +9,6 @@ package eu.etaxonomy.cdm.io.mexico; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -19,7 +18,6 @@ import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import eu.etaxonomy.cdm.common.URI; -import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport; import eu.etaxonomy.cdm.io.common.ResultSetPartitioner; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.reference.Reference; @@ -35,8 +33,6 @@ public class MexicoEfloraRefWebSitesImport extends MexicoEfloraReferenceImportBa private static final long serialVersionUID = -1186364983750790695L; private static final Logger logger = Logger.getLogger(MexicoEfloraRefWebSitesImport.class); - public static final String NAMESPACE = "WebSites"; - private static final String pluralString = "Websites"; private static final String dbTableName = "RefWebSites"; @@ -48,12 +44,8 @@ public class MexicoEfloraRefWebSitesImport extends MexicoEfloraReferenceImportBa public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) { boolean success = true ; - MexicoEfloraImportConfigurator config = state.getConfig(); Set refsToSave = new HashSet<>(); - @SuppressWarnings("unchecked") - Map refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE); - ResultSet rs = partitioner.getResultSet(); try{ while (rs.next()){ @@ -129,27 +121,7 @@ public class MexicoEfloraRefWebSitesImport extends MexicoEfloraReferenceImportBa @Override public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { - String nameSpace; - Set idSet; Map> result = new HashMap<>(); - - try{ - Set nameIdSet = new HashSet<>(); - Set referenceIdSet = new HashSet<>(); - while (rs.next()){ -// handleForeignKey(rs, nameIdSet, "PTNameFk"); -// handleForeignKey(rs, referenceIdSet, "PTRefFk"); - } - - //reference map - nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); - result.put(nameSpace, referenceMap); - - } catch (SQLException e) { - throw new RuntimeException(e); - } return result; } diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraReferenceImportBase.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraReferenceImportBase.java index f6ee7cfd..72386e5b 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraReferenceImportBase.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraReferenceImportBase.java @@ -65,10 +65,59 @@ public abstract class MexicoEfloraReferenceImportBase extends MexicoEfloraImpor //author if (isNotBlank(authorStr)) { + boolean isEds = false; + boolean isEd = false; + boolean isCoords = false; + boolean isCoord = false; + boolean isComps = false; + boolean isComp = false; + + if (authorStr.endsWith("(Eds.)") || authorStr.endsWith("(eds.)") + || authorStr.endsWith("(editors)")) { + isEds = true; + authorStr = authorStr.replace("(Eds.)", "").replace("(eds.)", "").replace("(editors)", "").trim(); + } + if (authorStr.endsWith("(Ed.)")) { + authorStr = authorStr.replace("(Ed.)", "").trim(); + isEd = true; + } + if (authorStr.endsWith("(Coords.)") || authorStr.endsWith("(coords.)")) { + authorStr = authorStr.replace("(Coords.)", "").replace("(coords.)", "").trim(); + isCoords = true; + } + if (authorStr.endsWith("(Coord.)") || authorStr.endsWith("(coord.)")) { + authorStr = authorStr.replace("(Coord.)", "").replace("(coord.)", "").trim(); + isCoords = true; + } + if (authorStr.endsWith("(Comps.)") || authorStr.endsWith("(comps.)")) { + authorStr = authorStr.replace("(comps.)", "").replace("(Comps.)", "").trim(); + isComps = true; + } + if (authorStr.endsWith("(Comp.)") || authorStr.endsWith("(comp.)")) { + authorStr = authorStr.replace("(comp.)", "").replace("(Comp.)", "").trim(); + isComps = true; + } + TeamOrPersonBase author = BibliographicAuthorParser .Instance().parse(authorStr); - //TODO is parsed really ok? - state.getDeduplicationHelper().getExistingAgent(author, true); + if (isEds) { + author.setTitleCache(author.getTitleCache()+ " (Eds.)", true); + }else if (isCoords) { + author.setTitleCache(author.getTitleCache()+ " (Coords.)", true); + }else if (isCoord) { + author.setTitleCache(author.getTitleCache()+ " (Coord.)", true); + }else if (isEd) { + author.setTitleCache(author.getTitleCache()+ " (Ed.)", true); + }else if (isComps) { + author.setTitleCache(author.getTitleCache()+ " (Comps.)", true); + }else if (isComp) { + author.setTitleCache(author.getTitleCache()+ " (Comp.)", true); + } + + + //not sure what is better, but seems to work with "false" + boolean parsed = false; + author = state.getDeduplicationHelper().getExistingAuthor(author, parsed); ref.setAuthorship(author); }else { logger.warn(refId + ": No author"); @@ -87,7 +136,7 @@ public abstract class MexicoEfloraReferenceImportBase extends MexicoEfloraImpor } protected void handleTitleStr(@SuppressWarnings("unused") MexicoEfloraImportState state, - String titleStr, Reference ref, int refId) { + String titleStr, Reference ref, @SuppressWarnings("unused") int refId) { //articleTitle if (isNotBlank(titleStr)) { @@ -130,12 +179,6 @@ public abstract class MexicoEfloraReferenceImportBase extends MexicoEfloraImpor protected void handleId(MexicoEfloraImportState state, int refId, Reference ref) { state.getReferenceUuidMap().put(refId, ref.getUuid()); -// state.getRefDetailMap().put(refId, detail); - - //TODO not needed anymore once "related objects" are adapted everywhere - Reference sourceRef = getSourceReference(state.getConfig().getSourceReference()); - ref.addImportSource(String.valueOf(refId), MexicoEfloraReferenceImportBase.NAMESPACE, - sourceRef, null); //.. identifier DefinedTerm conabioIdentifier = getIdentiferType(state, MexicoConabioTransformer.uuidConabioReferenceIdIdentifierType, diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRegionImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRegionImport.java index e9de6856..c50b5bd1 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRegionImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraRegionImport.java @@ -237,7 +237,6 @@ public class MexicoEfloraRegionImport extends MexicoEfloraImportBase { mapping.set(area, geoServiceArea); } - //not needed @Override public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraTaxonImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraTaxonImport.java index 773efdf8..a80f23ac 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraTaxonImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraTaxonImport.java @@ -13,9 +13,12 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -57,9 +60,8 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { private static final String pluralString = "Taxa"; protected static final String dbTableName = "EFlora_Taxonomia4CDM2"; - protected static INonViralNameParser nameParser = (INonViralNameParser)NonViralNameParserImpl.NewInstance(); - - + @SuppressWarnings({ "unchecked", "rawtypes" }) + protected static INonViralNameParser nameParser = (INonViralNameParser)NonViralNameParserImpl.NewInstance(); public MexicoEfloraTaxonImport(){ super(dbTableName, pluralString); @@ -69,7 +71,8 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { protected String getIdQuery(MexicoEfloraImportState state) { String sql = " SELECT IdCAT " + " FROM " + dbTableName - + " WHERE IdCAT NOT IN ('2PLANT','79217TRACH') " + + " WHERE IdCAT NOT IN ('2PLANT') " +// + " AND CitaNomenclatural LIKE 'Nov. Gen. Sp. Pl. (folio ed.) 4:%. 1820 [1818]' " + " ORDER BY IdCAT "; return sql; } @@ -92,6 +95,7 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { sourceReference = this.getSourceReference(state.getConfig().getSourceReference()); state.getDeduplicationHelper().reset(); + boolean success = true ; @SuppressWarnings("rawtypes") Set taxaToSave = new HashSet<>(); @@ -116,8 +120,8 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { return success; } - private boolean handleSingleRecord(ResultSetPartitioner partitioner, MexicoEfloraImportState state, boolean success, - Set taxaToSave, Map refMap, ResultSet rs, int i) throws SQLException { + private boolean handleSingleRecord(ResultSetPartitioner partitioner, MexicoEfloraImportState state, boolean success, + @SuppressWarnings("rawtypes") Set taxaToSave, Map refMap, ResultSet rs, int i) throws SQLException { if ((i % 1000) == 0 && i!= 1 ){ logger.info("Taxa handled: " + (i-1));} // System.out.println("i++"); //create Taxon element @@ -126,7 +130,6 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { String rankStr = rs.getString("CategoriaTaxonomica"); String nameStr = rs.getString("Nombre"); String autorStr = rs.getString("AutorSinAnio"); - String fullNameStr = nameStr + " " + autorStr; String citaNomenclaturalStr = rs.getString("CitaNomenclatural"); String annotationStr = rs.getString("AnotacionTaxon"); String type = rs.getString("NomPublicationType"); @@ -135,20 +138,6 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { UUID uuid = UUID.fromString(uuidStr); Integer secFk = nullSafeInt(rs, "IdBibliografiaSec"); - //name OLD handling -// Rank rank = getRank(rankStr); -// NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance(); -// TaxonName taxonName = (TaxonName)parser.parseFullName(fullNameStr, NomenclaturalCode.ICNAFP, rank); -// //.. identifier -// DefinedTerm conabioIdentifier = getIdentiferType(state, MexicoConabioTransformer.uuidConabioTaxonIdIdentifierType, -// "CONABIO Taxon Identifier", "CONABIO Taxon Identifier", "CONABIO", null); -// taxonName.addIdentifier(taxonId, conabioIdentifier); -// //.. nom Ref -// Reference nomRef = ReferenceFactory.newGeneric(); -// nomRef.setAbbrevTitleCache(citaNomenclaturalStr, true); -// nomRef.setDatePublished(TimePeriodParser.parseStringVerbatim(year)); -// taxonName.setNomenclaturalReference(nomRef); - TaxonName taxonName= makeName(taxonId, state, autorStr, nameStr, citaNomenclaturalStr, type, rankStr, annotationStr, year); @@ -158,9 +147,9 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { String refFkStr = String.valueOf(secFk); sec = refMap.get(refFkStr); if (sec == null && firstMissingSec) { - logger.warn("There are missing sec refs but they are not logged anymore."); - logger.debug("Sec not found for taxonId " + taxonId +" and secId " + refFkStr); - firstMissingSec = false; +// logger.warn("There are missing sec refs but they are not logged anymore."); + logger.warn("Sec not found for taxonId " + taxonId +" and secId " + refFkStr); + firstMissingSec = true; } } @@ -198,27 +187,24 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { //rank Rank rank = getRank(rankStr); - //TODO hybrido and race - boolean isHybrid = rank == null && "híbrido".equals(rankStr); + boolean isRace = Rank.RACE().equals(rank); -// rank = state.getTransformer().getRankByKey(rankStr); + String raceEpithet = null; + if (isRace) { + raceEpithet = nameStr.substring(nameStr.indexOf(" raza ") + 6).trim(); + nameStr = nameStr.substring(0, nameStr.indexOf(" raza ")).trim(); + } nameStr = removeSubgenusBracket(nameStr, rank); + nameStr = removeVarForSubvar(nameStr, rank); //name + author String fullNameStr = nameStr + (authorStr != null ? " " + authorStr : ""); TaxonName fullName = nameParser.parseFullName(fullNameStr, NomenclaturalCode.ICNAFP, rank); + if (fullName.isProtectedTitleCache()){ - logger.info(taxonId + ": Name could not be parsed: " + fullNameStr ); - }else{ - if (isFirstDedup) { - logger.warn("Deduplication is still switcht off!"); - //siehe auch weiter unten - isFirstDedup = false; - } - //FIXME dedup - state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(fullName); + logger.warn(taxonId + ": Name could not be parsed: " + fullNameStr ); } //reference @@ -229,27 +215,51 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { logger.warn(taxonId + ": Referenced name could not be parsed: " + refNameStr ); }else{ addSourcesToReferences(referencedName, state); - //FIXME deduplication -// state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(referencedName); } + if (isRace) { + //TODO Cultivar Race, shouldn't it be grex? + referencedName.setRank(Rank.GREX_ICNCP()); + referencedName.setNameType(NomenclaturalCode.ICNCP); + referencedName.setCultivarEpithet(raceEpithet); + } + adaptRefTypeForGeneric(referencedName, refType); Reference nomRef = referencedName.getNomenclaturalReference(); if (isNotBlank(year)) { + //if explicit year is given and year could not be parsed for nomRef we use the explicit year, + // otherwise if parsed date and explicit year differ it is logged if (nomRef == null) { nomRef = ReferenceFactory.newGeneric(); } String nomRefYear = nomRef.getYear(); if (isBlank(nomRefYear)) { + String nomRefDateStr = nomRef.getDatePublishedString(); nomRef.setDatePublished(TimePeriodParser.parseStringVerbatim(year)); + if (isNotBlank(nomRefDateStr) && !nomRefDateStr.equals(year)) { + Matcher matcher = Pattern.compile("([0-9]{4}(?:-[0-9]{4})?)\\s?\\[([0-9]{4})\\]").matcher(nomRefDateStr); + if (matcher.matches() && matcher.group(2).equals(year)){ + nomRef.getDatePublished().setVerbatimDate(matcher.group(1)); + logger.warn(taxonId + ": need to verify real year vs. verbatim year: "+ nomRef.getDatePublishedString() + " ("+referencedName.getTitleCache()+")"); + }else { + logger.warn(taxonId + ": year and parsed date published are not equal: "+ year + "<->" + nomRefDateStr + " ("+referencedName.getTitleCache()+")"); + } + } }else if (! nomRefYear.equals(year)){ - logger.warn(taxonId + ": year and parsed year are not equal: "+ year + "<->" + nomRefYear); + logger.warn(taxonId + ": year and parsed year are not equal: "+ year + "<->" + nomRefYear + " ("+referencedName.getTitleCache()+")"); } } + state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(referencedName); - TaxonName result= referencedName; + TaxonName result = referencedName; + state.getNameMap().put(result.getTitleCache(), result.getUuid()); + state.getNameMap().put(result.getNameCache(), result.getUuid()); //status - if (annotation != null && (annotation.equals("nom. illeg.") || annotation.equals("nom. cons."))){ + if (annotation != null && (annotation.equals("nom. illeg.") + || annotation.equals("nom. cons.") + || annotation.equals("nom. superfl.") + || annotation.equals("nom. inval.") + )){ try { NomenclaturalStatusType nomStatusType = NomenclaturalStatusType.getNomenclaturalStatusTypeByAbbreviation(annotation, result); result.addStatus(NomenclaturalStatus.NewInstance(nomStatusType)); @@ -258,7 +268,7 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { } } - if(result.getNomenclaturalReference()!=null && result.getNomenclaturalReference().getTitleCache().equals("null")){ + if(result.getNomenclaturalReference() != null && result.getNomenclaturalReference().getTitleCache().equals("null")){ logger.warn("null"); } @@ -269,7 +279,7 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { return result; } - private String removeSubgenusBracket(String nameStr, Rank rank) { + private String removeSubgenusBracket(String nameStr, @SuppressWarnings("unused") Rank rank) { if (nameStr.matches("[A-Z][a-z]+\\s+\\([A-Za-z]+\\)\\s+[a-z]+.*")) { //species and below: remove bracket completely nameStr = nameStr.substring(0, nameStr.indexOf("(")) + nameStr.substring(nameStr.indexOf(")")+1); @@ -280,6 +290,13 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { return nameStr; } + private String removeVarForSubvar(String nameStr, Rank rank) { + if (rank.equals(Rank.SUBVARIETY()) && nameStr.matches(".* var\\. .* subvar\\. .*")) { + nameStr = nameStr.substring(0, nameStr.indexOf(" var.")) + nameStr.substring(nameStr.indexOf(" subvar.")); + } + return nameStr; + } + private void adaptRefTypeForGeneric(IBotanicalName referencedName, String refTypeStr) { INomenclaturalReference ref = referencedName.getNomenclaturalReference(); if (ref == null){ @@ -328,7 +345,7 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { } } - protected IdentifiableSource makeOriginalSource(MexicoEfloraImportState state) { + protected IdentifiableSource makeOriginalSource(@SuppressWarnings("unused") MexicoEfloraImportState state) { return IdentifiableSource.NewDataImportInstance(null, null, sourceReference); } @@ -371,7 +388,6 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { public Map> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) { String nameSpace; - Set idSet; Map> result = new HashMap<>(); try{ @@ -380,11 +396,14 @@ public class MexicoEfloraTaxonImport extends MexicoEfloraImportBase { handleForeignKey(rs, referenceIdSet, "IdBibliografiaSec"); } - //reference map - nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE; - idSet = referenceIdSet; - Map referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace); - result.put(nameSpace, referenceMap); + //reference map + nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE; + Map referenceUuidMap = new HashMap<>(); + referenceIdSet.stream().forEach(rId->referenceUuidMap.put(state.getReferenceUuidMap().get(Integer.valueOf(rId)), rId)); + List references = getReferenceService().find(referenceUuidMap.keySet()); + Map referenceMap = new HashMap<>(); + references.stream().forEach(r->referenceMap.put(referenceUuidMap.get(r.getUuid()), r)); + result.put(nameSpace, referenceMap); } catch (SQLException e) { throw new RuntimeException(e); diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraTaxonRelationImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraTaxonRelationImport.java index 55953cf9..3a468311 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraTaxonRelationImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/mexico/MexicoEfloraTaxonRelationImport.java @@ -56,7 +56,7 @@ public class MexicoEfloraTaxonRelationImport extends MexicoEfloraImportBase { String sql = " SELECT IdCAT " + " FROM " + dbTableName + " t " + " LEFT JOIN cv1_Controlled_vocabulary_for_name_Ranks r ON t.CategoriaTaxonomica = r.NombreCategoriaTaxonomica " - + " WHERE t.IdCAT_AscendenteHerarquico4CDM NOT IN ('2PLANT','79217TRACH') " + + " WHERE t.IdCAT_AscendenteHerarquico4CDM NOT IN ('2PLANT') " + " ORDER BY r.Nivel1, IdCAT "; return sql; } @@ -105,14 +105,16 @@ public class MexicoEfloraTaxonRelationImport extends MexicoEfloraImportBase { TaxonBase taxonBase = taxonMap.get(uuid.toString()); try { - if (taxonBase.isInstanceOf(Synonym.class) && accUuidStr != null) { + if (taxonBase == null) { + logger.warn(taxonId + ": Taxon does not exist"); + continue; + }else if (taxonBase.isInstanceOf(Synonym.class) && accUuidStr != null) { Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class); TaxonBase related = taxonMap.get(accUuidStr); if (!related.isInstanceOf(Taxon.class)){ logger.warn(taxonId + ": Accepted taxon for synonym is not accepted: " + accUuidStr); }else { Taxon acc = CdmBase.deproxy(related, Taxon.class); - //TODO type acc.addSynonym(syn, SynonymType.SYNONYM_OF()); } }else if (taxonBase.isInstanceOf(Taxon.class) && parentUuidStr != null) { @@ -122,7 +124,6 @@ public class MexicoEfloraTaxonRelationImport extends MexicoEfloraImportBase { logger.warn(taxonId + ": Parent is not accepted: " + parentUuidStr); }else { Taxon parent = CdmBase.deproxy(parentBase, Taxon.class); - //TODO Reference parentChildReference = null; getClassification(state).addParentChild(parent, child, parentChildReference, null); } @@ -138,13 +139,13 @@ public class MexicoEfloraTaxonRelationImport extends MexicoEfloraImportBase { TaxonName name = taxonBase.getName(); TaxonBase basionymTaxon = taxonMap.get(basUuidStr); name.addBasionym(basionymTaxon.getName()); - //TODO synrel type } partitioner.startDoSave(); taxaToSave.add(taxonBase); } catch (Exception e) { logger.warn("An exception (" +e.getMessage()+") occurred when trying to create relation for id " + taxonId + ". Relation could not be saved."); + e.printStackTrace(); success = false; } } -- 2.34.1