static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_cdmtest_mysql();
+// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_test_mysql();
// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_test1();
// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_mexico_flora();
+ static final boolean doReferences = true;
//feature tree uuid
private static final UUID featureTreeUuid = UUID.fromString("dd2070ef-e75b-494b-bc6a-4a370271866b");
private static final UUID flatFeatureTreeUuid = UUID.fromString("33896d1f-c3df-4cdb-afb8-bffd35f9d6b0");
+ private static final UUID flatFeatureTreeWithoutUsesUuid = UUID.fromString("61832cbd-3800-4d43-8048-9514cc7b7f26");
private static final String featureTreeName = "Catalogo feature tree";
private static final String flatFeatureTreeName = "Catalogo flat feature tree";
+ private static final String flatFeatureTreeWithoutUsesName = "Catalogo flat feature tree without uses";
//classification
static final UUID classificationUuid = UUID.fromString("588dd2de-7c95-42ad-8eb1-994460561b5e");
- private static final String classificationName = "Plantae";
+ private static final String classificationName = "Plantas";
//check - import
static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
- boolean doVocabularies = (hbm2dll == DbSchemaValidation.CREATE);
-// static final boolean doTaxa = false;
-// static final boolean doDeduplicate = true;
-
protected void doImport(ICdmDataSource cdmDestination){
Source source = CdmImportSources.MEXICO_EFLORA();
config.setSecReference(getSecReference());
config.setFeatureTreeUuid(featureTreeUuid);
config.setFlatFeatureTreeUuid(flatFeatureTreeUuid);
+ config.setFlatFeatureTreeWithoutUsesUuid(flatFeatureTreeWithoutUsesUuid);
config.setFeatureTreeTitle(featureTreeName);
config.setFlatFeatureTreeTitle(flatFeatureTreeName);
+ config.setFlatFeatureTreeWithoutUsesTitle(flatFeatureTreeWithoutUsesName);
config.setRecordsPerTransaction(5000);
+ config.setDoReferences(doReferences);
config.setSource(source);
String fileName = source.toString();
public static final UUID uuidConabioTaxonIdIdentifierType = UUID.fromString("025fb8fe-e8e8-4136-bf37-93367a268889");
public static final UUID uuidConabioReferenceIdIdentifierType = UUID.fromString("a6f98411-b99b-45c5-9303-d2af421b9f73");
+ public static final UUID uuidConabioFactCategoryIdIdentifierType = UUID.fromString("8d6c1c38-2d23-40b2-8891-f74501d3763c");
public static final UUID uuidNomRefExtension = UUID.fromString("0a7e2f5f-c62d-43e1-874a-07cb1dbb9fa0");
ResultSet rs = state.getConfig().getSource().getResultSet(sql);
try {
while (rs.next()) {
- String lang = rs.getString("Lengua");
+ String lang = rs.getString("Lengua").trim();
Language language = Language.NewInstance(lang, lang, null, Language.SPANISH_CASTILIAN());
voc.addTerm(language);
getTermService().save(language); //not sure if necessary
}
}
- private TermVocabulary<Language> createLanguagesVoc(MexicoEfloraImportState state) {
+ private TermVocabulary<Language> createLanguagesVoc(@SuppressWarnings("unused") MexicoEfloraImportState state) {
URI termSourceUri = null;
String label = "Mexican Languages";
String description = "Mexican languages as used by the CONABIO database";
//create TaxonName element
String idCombi = rs.getString("IdCombinado");
- //TODO common name id
-// String idNomComun = rs.getString("IdNomComun");
+
+ String idNomComun = rs.getString("IdNomComun");
String taxonUuid = rs.getString("taxonUuid");
String nomComunStr = rs.getString("NomComun");
- String lenguaStr = rs.getString("Lengua");
+ String lenguaStr = rs.getString("Lengua").trim();
String paisStr = rs.getString("Pais");
String estadoStr = rs.getString("Estado");
NamedArea area = getArea(state, idRegion, estadoStr, paisStr, null, null);
CommonTaxonName commonName = CommonTaxonName.NewInstance(nomComunStr,
language, area);
- //TODO source reference correct (everywhere?)
+
TaxonDescription description = this.getTaxonDescription(taxon, sourceReference,
false, true);
description.addElement(commonName);
- state.getCommonNameMap().put(idCombi, commonName.getUuid());
+ commonName.addImportSource(idNomComun + ";" + idCombi, "Eflora_NombresComunes4CDM", sourceReference, null);
+ state.getCommonNameMap().put(idCombi, commonName.getUuid());
partitioner.startDoSave();
taxaToSave.add(taxonBase);
return success;
}
-
-
- private Language getLanguage(MexicoEfloraImportState state, String lenguaStr) {
+ private Language getLanguage(@SuppressWarnings("unused") MexicoEfloraImportState state, String lenguaStr) {
Language language = languageMap.get(lenguaStr);
if (isNotBlank(lenguaStr) && language == null) {
- logger.warn("Language not found: " + lenguaStr);
+ logger.warn("Language not found: '" + lenguaStr + "'");
}
return language;
}
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
try{
Set<UUID> taxonIdSet = new HashSet<>();
- Set<String> referenceIdSet = new HashSet<>();
while (rs.next()){
handleForeignUuidKey(rs, taxonIdSet, "taxonUuid");
-// handleForeignKey(rs, referenceIdSet, "PTRefFk");
}
//taxon map
taxa.stream().forEach(t->taxonMap.put(t.getUuid().toString(), t));
result.put(nameSpace, taxonMap);
- //reference map
- nameSpace = MexicoEfloraRefArticlesImport.NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
- result.put(nameSpace, referenceMap);
-
} catch (SQLException e) {
throw new RuntimeException(e);
}
import eu.etaxonomy.cdm.model.common.CdmBase;
import eu.etaxonomy.cdm.model.description.CommonTaxonName;
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
+import eu.etaxonomy.cdm.model.description.TaxonDescription;
import eu.etaxonomy.cdm.model.name.TaxonName;
import eu.etaxonomy.cdm.model.reference.Reference;
CommonTaxonName commonName = commonNameMap.get(idCombi);
Reference ref = referenceMap.get(String.valueOf(idBibliografia));
-// String detail = state.getRefDetailMap().get(idBibliografia);
String detail = null;
if (commonName != null) {
DescriptionElementSource source = commonName.addPrimaryTaxonomicSource(ref, detail);
if (source!= null) {
- TaxonName nameUsedInSource = getNameUsedInSource(state, observaciones);
+ TaxonName nameUsedInSource = getNameUsedInSource(state, observaciones, commonName, ref);
source.setNameUsedInSource(nameUsedInSource);
- //TODO other observaciones
} else {
logger.warn("Source not found for " + idCombi + " and bibID: " + idBibliografia);
}
return success;
}
- private TaxonName getNameUsedInSource(MexicoEfloraImportState state, String observaciones) {
- // TODO named used in source for common names
+ private TaxonName getNameUsedInSource(MexicoEfloraImportState state, String observaciones, CommonTaxonName commonName, Reference ref) {
+ if (observaciones != null) {
+ if (observaciones.matches("^(reportada |\\()?como .*")){
+ String nameStr = observaciones
+ .replaceAll("^(reportada |\\()?como ", "")
+ .replaceAll("\\)$", "");
+ UUID nameUuid = state.getNameMap().get(nameStr);
+ TaxonName name = getName(state, nameUuid);
+ if (name == null) {
+ String taxon = CdmBase.deproxy(commonName.getInDescription(),TaxonDescription.class).getTaxon().getName().getTitleCache();
+ logger.warn("Name in source ("+observaciones+") could not be found for " + taxon + "-" + commonName.getLanguage().getLabel() + "-" + commonName.getArea().getLabel() + "-" + ref.getTitleCache());
+ }
+ return name;
+ }
+ }
return null;
}
+ //quick and dirty and slow
+ private TaxonName getName(@SuppressWarnings("unused") MexicoEfloraImportState state, UUID nameUuid) {
+ return getNameService().find(nameUuid);
+ }
+
@Override
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
try{
//common name map
nameSpace = MexicoEfloraCommonNameImport.NAMESPACE;
- Map<UUID,String> uuidMap = new HashMap<>();
- commonNameIdSet.stream().forEach(cnId->uuidMap.put(state.getCommonNameMap().get(cnId),cnId));
+ Map<UUID,String> commonNameUuidMap = new HashMap<>();
+ commonNameIdSet.stream().forEach(cnId->commonNameUuidMap.put(state.getCommonNameMap().get(cnId), cnId));
@SuppressWarnings({ "rawtypes", "unchecked" })
- List<CommonTaxonName> commonNames = (List)getDescriptionElementService().find(uuidMap.keySet());
+ List<CommonTaxonName> commonNames = (List)getDescriptionElementService().find(commonNameUuidMap.keySet());
Map<String, CommonTaxonName> commonNameMap = new HashMap<>();
- commonNames.stream().forEach(cn->commonNameMap.put(uuidMap.get(cn.getUuid()), cn));
+ commonNames.stream().forEach(cn->commonNameMap.put(commonNameUuidMap.get(cn.getUuid()), cn));
result.put(nameSpace, commonNameMap);
- //reference map
+ //reference map
nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
+ Map<UUID,String> referenceUuidMap = new HashMap<>();
+ referenceIdSet.stream().forEach(rId->referenceUuidMap.put(state.getReferenceUuidMap().get(Integer.valueOf(rId)), rId));
+ List<Reference> references = getReferenceService().find(referenceUuidMap.keySet());
+ Map<String, Reference> referenceMap = new HashMap<>();
+ references.stream().forEach(r->referenceMap.put(referenceUuidMap.get(r.getUuid()), r));
result.put(nameSpace, referenceMap);
} catch (SQLException e) {
//create TaxonName element
String idCombi = rs.getString("IdDist");
+
String taxonUuid = rs.getString("taxonUuid");
- //TODO nombre in distribution gegenchecken
-// String nombreStr = rs.getString("Nombre");
+ String nombreStr = rs.getString("Nombre");
String paisStr = rs.getString("Pais");
String estadoStr = rs.getString("Estado");
String abreviaturaEstado = rs.getString("AbreviaturaEstado");
// int idTipoDistribucion = rs.getInt("IdTipoDistribucion");
int idTipoRegion = rs.getInt("IdTipoRegion");
- //FIXME mapping to mapservice
-
try {
TaxonBase<?> taxonBase = taxonMap.get(taxonUuid);
+ if(isNotBlank(nombreStr) && taxonBase != null && taxonBase.getName() != null
+ && !nombreStr.equals(taxonBase.getName().getNameCache())
+ && !nombreStr.contains("(") && !taxonBase.getName().isHybrid()) {
+ logger.warn(idCombi + ": Name differs " + nombreStr + "<->" + taxonBase.getName().getNameCache());
+ }
Taxon taxon;
if (taxonBase == null) {
logger.warn("Taxon "+taxonUuid+" not found for distribution " + idCombi);
Distribution distribution = Distribution.NewInstance(area, status);
- //TODO source reference correct (everywhere?)
TaxonDescription description = this.getTaxonDescription(
taxon, sourceReference, false, true);
description.addElement(distribution);
+ distribution.addImportSource(idCombi, "Eflora_DistribucionEstatalIndividual", sourceReference, null);
state.getDistributionMap().put(idCombi, distribution.getUuid());
partitioner.startDoSave();
import eu.etaxonomy.cdm.model.common.CdmBase;
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
import eu.etaxonomy.cdm.model.description.Distribution;
+import eu.etaxonomy.cdm.model.description.TaxonDescription;
import eu.etaxonomy.cdm.model.name.TaxonName;
import eu.etaxonomy.cdm.model.reference.Reference;
continue;
}
Reference ref = referenceMap.get(String.valueOf(idBibliografia));
-// String detail = state.getRefDetailMap().get(idBibliografia);
String detail = null;
DescriptionElementSource source = distribution.addPrimaryTaxonomicSource(ref, detail);
- //TODO nameUsedInSource for distribution
if (source!= null) {
- TaxonName nameUsedInSource = getNameUsedInSource(state, observaciones);
+ TaxonName nameUsedInSource = getNameUsedInSource(state, observaciones, distribution, ref);
source.setNameUsedInSource(nameUsedInSource);
- //TODO other observaciones
} else {
logger.warn("Source not found for " + idCombi + " and bibID: " + idBibliografia);
}
return success;
}
- private TaxonName getNameUsedInSource(MexicoEfloraImportState state, String observaciones) {
- // TODO Auto-generated method stub
+ private TaxonName getNameUsedInSource(MexicoEfloraImportState state, String observaciones, Distribution distribution, Reference ref) {
+ if (observaciones != null) {
+ if (observaciones.matches("^\\(?como .*")){
+ String nameStr = observaciones
+ .replaceAll("^\\(?como ", "")
+ .replaceAll("\\)$", "").trim();
+ if (nameStr.contains("registro obtenido a partir")) {
+ nameStr = nameStr.substring(0, nameStr.indexOf("registro obtenido a partir")).trim();
+ }
+
+ UUID nameUuid = state.getNameMap().get(nameStr);
+ TaxonName name = getName(state, nameUuid);
+ if (name == null) {
+ String taxon = CdmBase.deproxy(distribution.getInDescription(),TaxonDescription.class).getTaxon().getName().getTitleCache();
+ logger.warn("Name in source ("+observaciones+") could not be found for " + taxon + " - Area: " + distribution.getArea().getLabel() + " - Biblio: " + ref.getTitleCache());
+ }
+ return name;
+ }
+ }
return null;
}
+ //quick and dirty and slow
+ private TaxonName getName(@SuppressWarnings("unused") MexicoEfloraImportState state, UUID nameUuid) {
+ return getNameService().find(nameUuid);
+ }
+
@Override
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
try{
//distribution map
nameSpace = MexicoEfloraDistributionImport.NAMESPACE;
- Map<UUID,String> uuidMap = new HashMap<>();
- distributionIdSet.stream().forEach(dId->uuidMap.put(state.getDistributionMap().get(dId),dId));
+ Map<UUID,String> distributionUuidMap = new HashMap<>();
+ distributionIdSet.stream().forEach(dId->distributionUuidMap.put(state.getDistributionMap().get(dId),dId));
@SuppressWarnings({ "rawtypes", "unchecked" })
- List<Distribution> distributions = (List)getDescriptionElementService().find(uuidMap.keySet());
+ List<Distribution> distributions = (List)getDescriptionElementService().find(distributionUuidMap.keySet());
Map<String, Distribution> distributionMap = new HashMap<>();
- distributions.stream().forEach(d->distributionMap.put(uuidMap.get(d.getUuid()), d));
+ distributions.stream().forEach(d->distributionMap.put(distributionUuidMap.get(d.getUuid()), d));
result.put(nameSpace, distributionMap);
//reference map
nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
+ Map<UUID,String> referenceUuidMap = new HashMap<>();
+ referenceIdSet.stream().forEach(rId->referenceUuidMap.put(state.getReferenceUuidMap().get(Integer.valueOf(rId)), rId));
+ List<Reference> references = getReferenceService().find(referenceUuidMap.keySet());
+ Map<String, Reference> referenceMap = new HashMap<>();
+ references.stream().forEach(r->referenceMap.put(referenceUuidMap.get(r.getUuid()), r));
result.put(nameSpace, referenceMap);
} catch (SQLException e) {
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.springframework.transaction.TransactionStatus;
import eu.etaxonomy.cdm.common.UTF8;
-import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport;
-import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonNameImport;
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
import eu.etaxonomy.cdm.model.common.CdmBase;
import eu.etaxonomy.cdm.model.common.Language;
import eu.etaxonomy.cdm.model.description.Feature;
import eu.etaxonomy.cdm.model.description.State;
-import eu.etaxonomy.cdm.model.name.TaxonName;
-import eu.etaxonomy.cdm.model.reference.Reference;
import eu.etaxonomy.cdm.model.term.TermNode;
import eu.etaxonomy.cdm.model.term.TermTree;
import eu.etaxonomy.cdm.model.term.TermType;
//hierarchical feature tree
TermTree<Feature> featureTree = TermTree.NewFeatureInstance(state.getConfig().getFeatureTreeUuid());
featureTree.setLabel(state.getConfig().getFeatureTreeTitle(), Language.SPANISH_CASTILIAN());
- featureTree.setUuid(state.getConfig().getFeatureTreeUuid());
featureTree.getRoot().addChild(Feature.DISTRIBUTION());
getTermTreeService().save(featureTree);
//flat feature tree
- TermTree<Feature> flatFeatureTree = TermTree.NewFeatureInstance(state.getConfig().getFeatureTreeUuid());
+ TermTree<Feature> flatFeatureTree = TermTree.NewFeatureInstance(state.getConfig().getFlatFeatureTreeUuid());
flatFeatureTree.setLabel(state.getConfig().getFlatFeatureTreeTitle(), Language.SPANISH_CASTILIAN());
- flatFeatureTree.setUuid(state.getConfig().getFlatFeatureTreeUuid());
flatFeatureTree.getRoot().addChild(Feature.DISTRIBUTION());
getTermTreeService().save(flatFeatureTree);
+ //flat feature tree without Usos
+ TermTree<Feature> flatFeatureTreeWithoutUses = TermTree.NewFeatureInstance(state.getConfig().getFlatFeatureTreeWithoutUsesUuid());
+ flatFeatureTreeWithoutUses.setLabel(state.getConfig().getFlatFeatureTreeWithoutUsesTitle(), Language.SPANISH_CASTILIAN());
+ flatFeatureTreeWithoutUses.getRoot().addChild(Feature.DISTRIBUTION());
+ getTermTreeService().save(flatFeatureTreeWithoutUses);
getVocabularyService().save(featureVoc);
for (TreeNode child : root.children) {
- saveNodeRecursive(state, child, featureVoc, null, featureTree.getRoot(), flatFeatureTree.getRoot());
+ //filter out Uses
+ TermNode<Feature> withoutUsesRoot = child.idCatNombre == 460? null : flatFeatureTreeWithoutUses.getRoot();
+ saveNodeRecursive(state, child, featureVoc, null, featureTree.getRoot(), flatFeatureTree.getRoot(), withoutUsesRoot);
}
featureTree.getRoot().addChild(Feature.COMMON_NAME());
this.commitTransaction(tx);
}
- private void saveNodeRecursive(MexicoEfloraImportState state, TreeNode node, TermVocabulary<Feature> featureVoc, Feature parentFeature,
- TermNode<Feature> parentFeatureTreeNode, TermNode<Feature> flatFeatureTreereTreeNode) {
+ private void saveNodeRecursive(MexicoEfloraImportState state, TreeNode node,
+ TermVocabulary<Feature> featureVoc, Feature parentFeature,
+ TermNode<Feature> parentFeatureTreeNode, TermNode<Feature> flatFeatureTreeTreeNode,
+ TermNode<Feature> flatFeatureTreeWithoutUsesTreeNode) {
if (!node.children.isEmpty()) {
//is feature
label = label.startsWith(sep)? label.substring(sep.length()):label;
Feature feature = Feature.NewInstance(label, label, String.valueOf(node.idCatNombre), Language.SPANISH_CASTILIAN());
feature.setIdInVocabulary(node.key);
+ feature.addIdentifier(String.valueOf(node.idCatNombre), getIdentiferType(state, MexicoConabioTransformer.uuidConabioFactCategoryIdIdentifierType, "CONABIO Fact Identifier", "CONABIO Fact Identifier", null, null));
feature.setSupportsCategoricalData(true);
feature.setSupportsTextData(false);
featureVoc.addTerm(feature);
TermNode<Feature> featureTreeNode = parentFeatureTreeNode.addChild(feature);
- flatFeatureTreereTreeNode.addChild(feature);
+ flatFeatureTreeTreeNode.addChild(feature);
+ if (flatFeatureTreeWithoutUsesTreeNode != null) {
+ flatFeatureTreeWithoutUsesTreeNode.addChild(feature);
+ }
getTermService().save(feature);
//parent-child
if (parentFeature != null) {
parentFeature.addIncludes(feature);
}
for (TreeNode child : node.children) {
- saveNodeRecursive(state, child, featureVoc, feature, featureTreeNode, flatFeatureTreereTreeNode);
+ saveNodeRecursive(state, child, featureVoc, feature, featureTreeNode, flatFeatureTreeTreeNode, flatFeatureTreeWithoutUsesTreeNode);
}
}
// else {
}
State categoricalState = State.NewInstance(node.description, node.description, String.valueOf(node.idCatNombre), Language.SPANISH_CASTILIAN());
categoricalState.setIdInVocabulary(node.key);
+ categoricalState.addIdentifier(String.valueOf(node.idCatNombre), getIdentiferType(state, MexicoConabioTransformer.uuidConabioFactCategoryIdIdentifierType, "CONABIO Fact Identifier", "CONABIO Fact Identifier", null, null));
+
state.getStateMap().put(node.idCatNombre, categoricalState);
state.getFeatureMap().put(node.idCatNombre, parentFeature);
getTermService().save(categoricalState);
@Override
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
- String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
-
- try{
- Set<String> nameIdSet = new HashSet<>();
- Set<String> referenceIdSet = new HashSet<>();
- while (rs.next()){
-// handleForeignKey(rs, nameIdSet, "PTNameFk");
-// handleForeignKey(rs, referenceIdSet, "PTRefFk");
- }
-
- //name map
- nameSpace = BerlinModelTaxonNameImport.NAMESPACE;
- idSet = nameIdSet;
- Map<String, TaxonName> nameMap = getCommonService().getSourcedObjectsByIdInSourceC(TaxonName.class, idSet, nameSpace);
- result.put(nameSpace, nameMap);
-
- //reference map
- nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
- result.put(nameSpace, referenceMap);
-
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
return result;
}
@Override
protected String getIdQuery(MexicoEfloraImportState state) {
- String sql = " SELECT id "
- + " FROM " + dbTableName
- + " ORDER BY IdCAT, IdCatNombre, IdBibliografia ";
+ String sql = " SELECT f.id "
+ + " FROM " + dbTableName + " f "
+ + " LEFT JOIN Eflora_CatalogoNombre4CDM c ON c.IdCatNombre = f.IdCatNombre"
+ + " ORDER BY f.IdCAT, c.Nivel1, c.Nivel2, c.Nivel3, c.Nivel4, c.Nivel5, c.Nivel6, c.Nivel7, f.IdCatNombre, f.IdBibliografia ";
return sql;
}
private CategoricalData lastFact;
private String lastIdCat = "-1";
private int lastIdCatNombre = -1;
+ @SuppressWarnings("unused")
+ private int lastBiblioId = -1;
+
@Override
public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) {
String uuidTaxonStr = rs.getString("taxonUuid");
int idBibliografia = rs.getInt("IdBibliografia");
- //TODO observaciones in facts
- String observaciones = rs.getString("Observaciones");
+ //handled in Excel log
+// String observaciones = rs.getString("Observaciones");
try {
CategoricalData categoricalData;
if (idCAT.equals(lastIdCat) && idCatNombre == lastIdCatNombre) {
+ //same taxon, same feature, same state => only add bibliography
categoricalData = lastFact;
}else {
+ //create to potential categorical data
categoricalData = makeCategoricalData(state, idCatNombre);
Feature lastFeature = lastFact == null? null : lastFact.getFeature();
if (idCAT.equals(lastIdCat) && categoricalData.getFeature().equals(lastFeature)) {
//add the single new state to the existing categorical data
//TODO not fully correct if bibliography differs for the single states;
State newState = categoricalData.getStatesOnly().stream().findFirst().orElse(null);
- if (newState != null) {
+ if (newState != null && !lastFact.hasState(newState)) {
lastFact.addStateData(newState);
}
categoricalData = lastFact;
-// lastIdCatNombre = idCatNombre;
+ lastIdCatNombre = idCatNombre;
}else {
//new categorical data
TaxonBase<?> taxonBase = taxonMap.get(uuidTaxonStr);
continue;
}
- //TODO source reference correct?
TaxonDescription description = this.getTaxonDescription(taxon, sourceReference,
false, true);
+
description.addElement(categoricalData);
- lastFact = categoricalData;
- lastIdCat = idCAT;
- lastIdCatNombre = idCatNombre;
+ categoricalData.addImportSource(idCAT+";"+idCatNombre, "Eflora_RelBiblioNombreCatalogoNombre", sourceReference, null);
}
}
handleBibliografia(state, referenceMap, categoricalData, idBibliografia, id);
+ lastBiblioId = idBibliografia;
+ lastIdCat = idCAT;
+ lastIdCatNombre = idCatNombre;
+ lastFact = categoricalData;
partitioner.startDoSave();
} catch (Exception e) {
return success;
}
- private void handleBibliografia(MexicoEfloraImportState state, Map<String, Reference> referenceMap,
+ private void handleBibliografia(@SuppressWarnings("unused") MexicoEfloraImportState state, Map<String, Reference> referenceMap,
CategoricalData categoricalData, int idBibliografia,
int id) {
Reference ref = referenceMap == null ? null : referenceMap.get(String.valueOf(idBibliografia));
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
try{
taxa.stream().forEach(t->taxonMap.put(t.getUuid().toString(), t));
result.put(nameSpace, taxonMap);
- //reference map
- nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
- result.put(nameSpace, referenceMap);
+ //reference map
+ nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE;
+ Map<UUID,String> referenceUuidMap = new HashMap<>();
+ referenceIdSet.stream().forEach(rId->referenceUuidMap.put(state.getReferenceUuidMap().get(Integer.valueOf(rId)), rId));
+ List<Reference> references = getReferenceService().find(referenceUuidMap.keySet());
+ Map<String, Reference> referenceMap = new HashMap<>();
+ references.stream().forEach(r->referenceMap.put(referenceUuidMap.get(r.getUuid()), r));
+ result.put(nameSpace, referenceMap);
} catch (SQLException e) {
throw new RuntimeException(e);
String estadoStr, String paisStr, String abbrev, Integer idTipoRegion) {
NamedArea areaById = state.getAreaMap().get(idRegion);
- if (areaIsMexicoCountry(idRegion, estadoStr, paisStr, idTipoRegion)) {
+ if (areaIsMexicoCountry(idRegion, estadoStr, paisStr)) {
return areaById;
}
String label = isNotBlank(estadoStr) ? estadoStr : paisStr;
logger.warn("Area not found, create new one: " + idRegion + "; " + label);
NamedAreaLevel level = idTipoRegion != null && idTipoRegion.equals(1)? NamedAreaLevel.COUNTRY() : null;
NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
- //TODO new namedAreas vocabulary
NamedArea namedArea = this.getNamedArea(state, null, label, label, abbrev, areaType, level, null, null);
state.getAreaLabelMap().put(labelKey, namedArea);
}
return result;
}
- private NamedArea getNewCountry(MexicoEfloraImportState state, String label) {
+ private NamedArea getNewCountry(@SuppressWarnings("unused") MexicoEfloraImportState state, String label) {
if (StringUtils.isBlank(label)) {
return null;
}else if (label.equalsIgnoreCase("CANADÁ")) {return Country.CANADA();
}else if (label.equalsIgnoreCase("ANGOLA")) {return Country.ANGOLAREPUBLICOF();
}else if (label.equalsIgnoreCase("TAILANDIA")) {return Country.THAILANDKINGDOMOF();
}else if (label.equalsIgnoreCase("TANZANIA")) {return Country.TANZANIAUNITEDREPUBLICOF();
+ }else if (label.equalsIgnoreCase("ARGENTINA")) {return Country.ARGENTINAARGENTINEREPUBLIC();
+ }else if (label.equalsIgnoreCase("SURINAM")) {return Country.SURINAMEREPUBLICOF();
+ }else if (label.equalsIgnoreCase("INGLATERRA")) {return Country.UNITEDKINGDOMOFGREATBRITAINANDNORTHERNIRELAND();
}
return null;
}
private boolean areaIsMexicoCountry(Integer idRegion, String estadoStr,
- String paisStr, Integer idTipoRegion) {
+ String paisStr) {
return idRegion == 2 && isBlank(estadoStr) &&
- "MÉXICO".equalsIgnoreCase(paisStr); // && idTipoRegion == 1;
+ "MÉXICO".equalsIgnoreCase(paisStr);
}
}
\ No newline at end of file
return new MexicoEfloraImportConfigurator(berlinModelSource, destination);
}
- //TODO
private static IInputTransformer defaultTransformer = new MexicoConabioTransformer();
private boolean doCommonNames = true;
private UUID featureTreeUuid;
private UUID flatFeatureTreeUuid;
+ private UUID flatFeatureTreeWithoutUsesUuid;
private String featureTreeTitle;
private String flatFeatureTreeTitle;
-
+ private String flatFeatureTreeWithoutUsesTitle;
private Reference secReference;
super(berlinModelSource, destination, NomenclaturalCode.ICNAFP, defaultTransformer); //default for Berlin Model
}
-
public boolean isDoCommonNames() {
return doCommonNames;
}
public void setDoCommonNames(boolean doCommonNames) {
this.doCommonNames = doCommonNames;
-
}
public boolean isDoFacts() {
this.flatFeatureTreeUuid = flatFeatureTreeUuid;
}
+ public UUID getFlatFeatureTreeWithoutUsesUuid() {
+ return flatFeatureTreeWithoutUsesUuid;
+ }
+ public void setFlatFeatureTreeWithoutUsesUuid(UUID flatFeatureTreeWithoutUsesUuid) {
+ this.flatFeatureTreeWithoutUsesUuid = flatFeatureTreeWithoutUsesUuid;
+ }
+
@Override
public String getFeatureTreeTitle() {
return featureTreeTitle;
this.flatFeatureTreeTitle = flatFeatureTreeTitle;
}
+ public String getFlatFeatureTreeWithoutUsesTitle() {
+ return flatFeatureTreeWithoutUsesTitle;
+ }
+ public void setFlatFeatureTreeWithoutUsesTitle(String flatFeatureTreeWithoutUsesTitle) {
+ this.flatFeatureTreeWithoutUsesTitle = flatFeatureTreeWithoutUsesTitle;
+ }
+
public boolean isDoNamedAreas() {
return doNamedAreas;
}
this.secReference = secReference;
}
+
}
@SuppressWarnings("unused")
private static final Logger logger = Logger.getLogger(MexicoEfloraImportState.class);
- private Map<String, DefinedTermBase> dbCdmDefTermMap = new HashMap<>();
+ private Map<String, DefinedTermBase<?>> dbCdmDefTermMap = new HashMap<>();
private Map<Integer,Feature> featureMap = new HashMap<>();
private Map<Integer,State> stateMap = new HashMap<>();
private Map<String,NamedArea> areaLabelMap = new HashMap<>();
private Map<Integer,UUID> referenceUuidMap = new HashMap<>();
-// private Map<Integer,String> refDetailMap = new HashMap<>();
private Map<String,UUID> commonNameMap = new HashMap<>();
private Map<String,UUID> distributionMap = new HashMap<>();
+ private Map<String,UUID> nameMap = new HashMap<>();
+
public MexicoEfloraImportState(MexicoEfloraImportConfigurator config) {
super(config);
}
- public Map<String, DefinedTermBase> getDbCdmDefinedTermMap(){
+ public Map<String, DefinedTermBase<?>> getDbCdmDefinedTermMap(){
return this.dbCdmDefTermMap;
}
- public void putDefinedTermToMap(String tableName, String id, DefinedTermBase term){
+ public void putDefinedTermToMap(String tableName, String id, DefinedTermBase<?> term){
this.dbCdmDefTermMap.put(tableName + "_" + id, term);
}
- public void putDefinedTermToMap(String tableName, int id, DefinedTermBase term){
+ public void putDefinedTermToMap(String tableName, int id, DefinedTermBase<?> term){
putDefinedTermToMap(tableName, String.valueOf(id), term);
}
public Map<Integer,UUID> getReferenceUuidMap() {
return referenceUuidMap;
}
-// public Map<Integer,String> getRefDetailMap() {
-// return refDetailMap;
-// }
public Map<String,UUID> getCommonNameMap() {
return commonNameMap;
}
-
+ public Map<String,UUID> getNameMap() {
+ return nameMap;
+ }
}
package eu.etaxonomy.cdm.io.mexico;
import java.sql.ResultSet;
-import java.sql.SQLException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
-import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport;
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
import eu.etaxonomy.cdm.model.common.CdmBase;
import eu.etaxonomy.cdm.model.reference.IJournal;
private static final Logger logger = Logger.getLogger(MexicoEfloraRefArticlesImport.class);
- public static final String NAMESPACE = "Articles";
-
private static final String pluralString = "Articles";
private static final String dbTableName = "RefArticles";
private static final Map<String,IJournal> journalMap = new HashMap<>();
public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) {
boolean success = true ;
- MexicoEfloraImportConfigurator config = state.getConfig();
Set<Reference> refsToSave = new HashSet<>();
- @SuppressWarnings("unchecked")
- Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
-
ResultSet rs = partitioner.getResultSet();
try{
while (rs.next()){
String articleTitleStr = rs.getString("ArticleTitle");
String journalTitleStr = rs.getString("JournalTitle");
String concat = rs.getString("Concatenation");
+ //TODO _minor V and P in articles
String vStr = rs.getString("V");
String pStr = rs.getString("P");
String urlStr = rs.getString("URL");
@Override
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
- String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
-
- try{
- Set<String> nameIdSet = new HashSet<>();
- Set<String> referenceIdSet = new HashSet<>();
- while (rs.next()){
-// handleForeignKey(rs, nameIdSet, "PTNameFk");
-// handleForeignKey(rs, referenceIdSet, "PTRefFk");
- }
-
- //reference map
- nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
- result.put(nameSpace, referenceMap);
-
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
return result;
}
package eu.etaxonomy.cdm.io.mexico;
import java.sql.ResultSet;
-import java.sql.SQLException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
-import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport;
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
import eu.etaxonomy.cdm.model.common.CdmBase;
import eu.etaxonomy.cdm.model.reference.Reference;
private static final long serialVersionUID = -1186364983750790695L;
private static final Logger logger = Logger.getLogger(MexicoEfloraRefOtherBooksImport.class);
- public static final String NAMESPACE = "OtherBooks";
-
private static final String pluralString = "Other books";
private static final String dbTableName = "RefOtherBooks";
public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) {
boolean success = true ;
- MexicoEfloraImportConfigurator config = state.getConfig();
- Set<Reference> refsToSave = new HashSet<>();
-
- @SuppressWarnings("unchecked")
- Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
+ Set<Reference> refsToSave = new HashSet<>();
ResultSet rs = partitioner.getResultSet();
try{
String titleStr = rs.getString("Title");
String volumeStr = rs.getString("Volume");
String pagesStr = rs.getString("Pages");
- String observacionesStr = rs.getString("Observaciones");
+ //exported as Excel table
+// String observacionesStr = rs.getString("Observaciones");
String urlStr = rs.getString("URL");
String doiStr = rs.getString("DOI");
String isbnStr = rs.getString("ISBN");
//title
handleTitleStr(state, titleStr, ref, refId);
+ //pages
+ if (isNotBlank(pagesStr)) {
+ ref.setPages(pagesStr);
+ }
+
//concat
if (isNotBlank(volumeStr)) {
ref.setVolume(volumeStr);
@Override
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
- String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
-
- try{
- Set<String> nameIdSet = new HashSet<>();
- Set<String> referenceIdSet = new HashSet<>();
- while (rs.next()){
-// handleForeignKey(rs, nameIdSet, "PTNameFk");
-// handleForeignKey(rs, referenceIdSet, "PTRefFk");
- }
-
- //reference map
- nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
- result.put(nameSpace, referenceMap);
-
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
return result;
}
package eu.etaxonomy.cdm.io.mexico;
import java.sql.ResultSet;
-import java.sql.SQLException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
-import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport;
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
import eu.etaxonomy.cdm.model.common.CdmBase;
import eu.etaxonomy.cdm.model.reference.Reference;
public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) {
boolean success = true ;
- MexicoEfloraImportConfigurator config = state.getConfig();
- Set<Reference> refsToSave = new HashSet<>();
-
- @SuppressWarnings("unchecked")
- Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
+ Set<Reference> refsToSave = new HashSet<>();
ResultSet rs = partitioner.getResultSet();
try{
String titleStr = rs.getString("Title");
String volumeStr = rs.getString("Volume");
String pagesStr = rs.getString("Pages");
- String observacionesStr = rs.getString("Observaciones");
+ //exported as Excel table
+// String observacionesStr = rs.getString("Observaciones");
String urlStr = rs.getString("URL");
String doiStr = rs.getString("DOI");
String isbnStr = rs.getString("ISBN");
//title
handleTitleStr(state, titleStr, ref, refId);
+ //pages
+ if (isNotBlank(pagesStr)) {
+ ref.setPages(pagesStr);
+ }
+
//concat
if (isNotBlank(volumeStr)) {
ref.setVolume(volumeStr);
@Override
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
- String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
-
- try{
- Set<String> nameIdSet = new HashSet<>();
- Set<String> referenceIdSet = new HashSet<>();
- while (rs.next()){
-// handleForeignKey(rs, nameIdSet, "PTNameFk");
-// handleForeignKey(rs, referenceIdSet, "PTRefFk");
- }
-
- //reference map
- nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
- result.put(nameSpace, referenceMap);
-
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
return result;
}
package eu.etaxonomy.cdm.io.mexico;
import java.sql.ResultSet;
-import java.sql.SQLException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import org.springframework.stereotype.Component;
import eu.etaxonomy.cdm.common.URI;
-import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport;
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
import eu.etaxonomy.cdm.model.common.CdmBase;
import eu.etaxonomy.cdm.model.reference.Reference;
private static final long serialVersionUID = -1186364983750790695L;
private static final Logger logger = Logger.getLogger(MexicoEfloraRefWebSitesImport.class);
- public static final String NAMESPACE = "WebSites";
-
private static final String pluralString = "Websites";
private static final String dbTableName = "RefWebSites";
public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, MexicoEfloraImportState state) {
boolean success = true ;
- MexicoEfloraImportConfigurator config = state.getConfig();
Set<Reference> refsToSave = new HashSet<>();
- @SuppressWarnings("unchecked")
- Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
-
ResultSet rs = partitioner.getResultSet();
try{
while (rs.next()){
@Override
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
- String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
-
- try{
- Set<String> nameIdSet = new HashSet<>();
- Set<String> referenceIdSet = new HashSet<>();
- while (rs.next()){
-// handleForeignKey(rs, nameIdSet, "PTNameFk");
-// handleForeignKey(rs, referenceIdSet, "PTRefFk");
- }
-
- //reference map
- nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
- result.put(nameSpace, referenceMap);
-
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
return result;
}
//author
if (isNotBlank(authorStr)) {
+ boolean isEds = false;
+ boolean isEd = false;
+ boolean isCoords = false;
+ boolean isCoord = false;
+ boolean isComps = false;
+ boolean isComp = false;
+
+ if (authorStr.endsWith("(Eds.)") || authorStr.endsWith("(eds.)")
+ || authorStr.endsWith("(editors)")) {
+ isEds = true;
+ authorStr = authorStr.replace("(Eds.)", "").replace("(eds.)", "").replace("(editors)", "").trim();
+ }
+ if (authorStr.endsWith("(Ed.)")) {
+ authorStr = authorStr.replace("(Ed.)", "").trim();
+ isEd = true;
+ }
+ if (authorStr.endsWith("(Coords.)") || authorStr.endsWith("(coords.)")) {
+ authorStr = authorStr.replace("(Coords.)", "").replace("(coords.)", "").trim();
+ isCoords = true;
+ }
+ if (authorStr.endsWith("(Coord.)") || authorStr.endsWith("(coord.)")) {
+ authorStr = authorStr.replace("(Coord.)", "").replace("(coord.)", "").trim();
+ isCoords = true;
+ }
+ if (authorStr.endsWith("(Comps.)") || authorStr.endsWith("(comps.)")) {
+ authorStr = authorStr.replace("(comps.)", "").replace("(Comps.)", "").trim();
+ isComps = true;
+ }
+ if (authorStr.endsWith("(Comp.)") || authorStr.endsWith("(comp.)")) {
+ authorStr = authorStr.replace("(comp.)", "").replace("(Comp.)", "").trim();
+ isComps = true;
+ }
+
TeamOrPersonBase<?> author = BibliographicAuthorParser
.Instance().parse(authorStr);
- //TODO is parsed really ok?
- state.getDeduplicationHelper().getExistingAgent(author, true);
+ if (isEds) {
+ author.setTitleCache(author.getTitleCache()+ " (Eds.)", true);
+ }else if (isCoords) {
+ author.setTitleCache(author.getTitleCache()+ " (Coords.)", true);
+ }else if (isCoord) {
+ author.setTitleCache(author.getTitleCache()+ " (Coord.)", true);
+ }else if (isEd) {
+ author.setTitleCache(author.getTitleCache()+ " (Ed.)", true);
+ }else if (isComps) {
+ author.setTitleCache(author.getTitleCache()+ " (Comps.)", true);
+ }else if (isComp) {
+ author.setTitleCache(author.getTitleCache()+ " (Comp.)", true);
+ }
+
+
+ //not sure what is better, but seems to work with "false"
+ boolean parsed = false;
+ author = state.getDeduplicationHelper().getExistingAuthor(author, parsed);
ref.setAuthorship(author);
}else {
logger.warn(refId + ": No author");
}
protected void handleTitleStr(@SuppressWarnings("unused") MexicoEfloraImportState state,
- String titleStr, Reference ref, int refId) {
+ String titleStr, Reference ref, @SuppressWarnings("unused") int refId) {
//articleTitle
if (isNotBlank(titleStr)) {
protected void handleId(MexicoEfloraImportState state, int refId, Reference ref) {
state.getReferenceUuidMap().put(refId, ref.getUuid());
-// state.getRefDetailMap().put(refId, detail);
-
- //TODO not needed anymore once "related objects" are adapted everywhere
- Reference sourceRef = getSourceReference(state.getConfig().getSourceReference());
- ref.addImportSource(String.valueOf(refId), MexicoEfloraReferenceImportBase.NAMESPACE,
- sourceRef, null);
//.. identifier
DefinedTerm conabioIdentifier = getIdentiferType(state, MexicoConabioTransformer.uuidConabioReferenceIdIdentifierType,
mapping.set(area, geoServiceArea);
}
-
//not needed
@Override
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
import java.sql.SQLException;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
private static final String pluralString = "Taxa";
protected static final String dbTableName = "EFlora_Taxonomia4CDM2";
- protected static INonViralNameParser<TaxonName> nameParser = (INonViralNameParser)NonViralNameParserImpl.NewInstance();
-
-
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ protected static INonViralNameParser<TaxonName> nameParser = (INonViralNameParser)NonViralNameParserImpl.NewInstance();
public MexicoEfloraTaxonImport(){
super(dbTableName, pluralString);
protected String getIdQuery(MexicoEfloraImportState state) {
String sql = " SELECT IdCAT "
+ " FROM " + dbTableName
- + " WHERE IdCAT NOT IN ('2PLANT','79217TRACH') "
+ + " WHERE IdCAT NOT IN ('2PLANT') "
+// + " AND CitaNomenclatural LIKE 'Nov. Gen. Sp. Pl. (folio ed.) 4:%. 1820 [1818]' "
+ " ORDER BY IdCAT ";
return sql;
}
sourceReference = this.getSourceReference(state.getConfig().getSourceReference());
state.getDeduplicationHelper().reset();
+
boolean success = true ;
@SuppressWarnings("rawtypes")
Set<TaxonBase> taxaToSave = new HashSet<>();
return success;
}
- private boolean handleSingleRecord(ResultSetPartitioner partitioner, MexicoEfloraImportState state, boolean success,
- Set<TaxonBase> taxaToSave, Map<String, Reference> refMap, ResultSet rs, int i) throws SQLException {
+ private boolean handleSingleRecord(ResultSetPartitioner<?> partitioner, MexicoEfloraImportState state, boolean success,
+ @SuppressWarnings("rawtypes") Set<TaxonBase> taxaToSave, Map<String, Reference> refMap, ResultSet rs, int i) throws SQLException {
if ((i % 1000) == 0 && i!= 1 ){ logger.info("Taxa handled: " + (i-1));}
// System.out.println("i++");
//create Taxon element
String rankStr = rs.getString("CategoriaTaxonomica");
String nameStr = rs.getString("Nombre");
String autorStr = rs.getString("AutorSinAnio");
- String fullNameStr = nameStr + " " + autorStr;
String citaNomenclaturalStr = rs.getString("CitaNomenclatural");
String annotationStr = rs.getString("AnotacionTaxon");
String type = rs.getString("NomPublicationType");
UUID uuid = UUID.fromString(uuidStr);
Integer secFk = nullSafeInt(rs, "IdBibliografiaSec");
- //name OLD handling
-// Rank rank = getRank(rankStr);
-// NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
-// TaxonName taxonName = (TaxonName)parser.parseFullName(fullNameStr, NomenclaturalCode.ICNAFP, rank);
-// //.. identifier
-// DefinedTerm conabioIdentifier = getIdentiferType(state, MexicoConabioTransformer.uuidConabioTaxonIdIdentifierType,
-// "CONABIO Taxon Identifier", "CONABIO Taxon Identifier", "CONABIO", null);
-// taxonName.addIdentifier(taxonId, conabioIdentifier);
-// //.. nom Ref
-// Reference nomRef = ReferenceFactory.newGeneric();
-// nomRef.setAbbrevTitleCache(citaNomenclaturalStr, true);
-// nomRef.setDatePublished(TimePeriodParser.parseStringVerbatim(year));
-// taxonName.setNomenclaturalReference(nomRef);
-
TaxonName taxonName= makeName(taxonId, state, autorStr,
nameStr, citaNomenclaturalStr, type, rankStr, annotationStr, year);
String refFkStr = String.valueOf(secFk);
sec = refMap.get(refFkStr);
if (sec == null && firstMissingSec) {
- logger.warn("There are missing sec refs but they are not logged anymore.");
- logger.debug("Sec not found for taxonId " + taxonId +" and secId " + refFkStr);
- firstMissingSec = false;
+// logger.warn("There are missing sec refs but they are not logged anymore.");
+ logger.warn("Sec not found for taxonId " + taxonId +" and secId " + refFkStr);
+ firstMissingSec = true;
}
}
//rank
Rank rank = getRank(rankStr);
- //TODO hybrido and race
- boolean isHybrid = rank == null && "híbrido".equals(rankStr);
+
boolean isRace = Rank.RACE().equals(rank);
-// rank = state.getTransformer().getRankByKey(rankStr);
+ String raceEpithet = null;
+ if (isRace) {
+ raceEpithet = nameStr.substring(nameStr.indexOf(" raza ") + 6).trim();
+ nameStr = nameStr.substring(0, nameStr.indexOf(" raza ")).trim();
+ }
nameStr = removeSubgenusBracket(nameStr, rank);
+ nameStr = removeVarForSubvar(nameStr, rank);
//name + author
String fullNameStr = nameStr + (authorStr != null ? " " + authorStr : "");
TaxonName fullName = nameParser.parseFullName(fullNameStr, NomenclaturalCode.ICNAFP, rank);
+
if (fullName.isProtectedTitleCache()){
- logger.info(taxonId + ": Name could not be parsed: " + fullNameStr );
- }else{
- if (isFirstDedup) {
- logger.warn("Deduplication is still switcht off!");
- //siehe auch weiter unten
- isFirstDedup = false;
- }
- //FIXME dedup
- state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(fullName);
+ logger.warn(taxonId + ": Name could not be parsed: " + fullNameStr );
}
//reference
logger.warn(taxonId + ": Referenced name could not be parsed: " + refNameStr );
}else{
addSourcesToReferences(referencedName, state);
- //FIXME deduplication
-// state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(referencedName);
}
+ if (isRace) {
+ //TODO Cultivar Race, shouldn't it be grex?
+ referencedName.setRank(Rank.GREX_ICNCP());
+ referencedName.setNameType(NomenclaturalCode.ICNCP);
+ referencedName.setCultivarEpithet(raceEpithet);
+ }
+
adaptRefTypeForGeneric(referencedName, refType);
Reference nomRef = referencedName.getNomenclaturalReference();
if (isNotBlank(year)) {
+ //if explicit year is given and year could not be parsed for nomRef we use the explicit year,
+ // otherwise if parsed date and explicit year differ it is logged
if (nomRef == null) {
nomRef = ReferenceFactory.newGeneric();
}
String nomRefYear = nomRef.getYear();
if (isBlank(nomRefYear)) {
+ String nomRefDateStr = nomRef.getDatePublishedString();
nomRef.setDatePublished(TimePeriodParser.parseStringVerbatim(year));
+ if (isNotBlank(nomRefDateStr) && !nomRefDateStr.equals(year)) {
+ Matcher matcher = Pattern.compile("([0-9]{4}(?:-[0-9]{4})?)\\s?\\[([0-9]{4})\\]").matcher(nomRefDateStr);
+ if (matcher.matches() && matcher.group(2).equals(year)){
+ nomRef.getDatePublished().setVerbatimDate(matcher.group(1));
+ logger.warn(taxonId + ": need to verify real year vs. verbatim year: "+ nomRef.getDatePublishedString() + " ("+referencedName.getTitleCache()+")");
+ }else {
+ logger.warn(taxonId + ": year and parsed date published are not equal: "+ year + "<->" + nomRefDateStr + " ("+referencedName.getTitleCache()+")");
+ }
+ }
}else if (! nomRefYear.equals(year)){
- logger.warn(taxonId + ": year and parsed year are not equal: "+ year + "<->" + nomRefYear);
+ logger.warn(taxonId + ": year and parsed year are not equal: "+ year + "<->" + nomRefYear + " ("+referencedName.getTitleCache()+")");
}
}
+ state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(referencedName);
- TaxonName result= referencedName;
+ TaxonName result = referencedName;
+ state.getNameMap().put(result.getTitleCache(), result.getUuid());
+ state.getNameMap().put(result.getNameCache(), result.getUuid());
//status
- if (annotation != null && (annotation.equals("nom. illeg.") || annotation.equals("nom. cons."))){
+ if (annotation != null && (annotation.equals("nom. illeg.")
+ || annotation.equals("nom. cons.")
+ || annotation.equals("nom. superfl.")
+ || annotation.equals("nom. inval.")
+ )){
try {
NomenclaturalStatusType nomStatusType = NomenclaturalStatusType.getNomenclaturalStatusTypeByAbbreviation(annotation, result);
result.addStatus(NomenclaturalStatus.NewInstance(nomStatusType));
}
}
- if(result.getNomenclaturalReference()!=null && result.getNomenclaturalReference().getTitleCache().equals("null")){
+ if(result.getNomenclaturalReference() != null && result.getNomenclaturalReference().getTitleCache().equals("null")){
logger.warn("null");
}
return result;
}
- private String removeSubgenusBracket(String nameStr, Rank rank) {
+ private String removeSubgenusBracket(String nameStr, @SuppressWarnings("unused") Rank rank) {
if (nameStr.matches("[A-Z][a-z]+\\s+\\([A-Za-z]+\\)\\s+[a-z]+.*")) {
//species and below: remove bracket completely
nameStr = nameStr.substring(0, nameStr.indexOf("(")) + nameStr.substring(nameStr.indexOf(")")+1);
return nameStr;
}
+ private String removeVarForSubvar(String nameStr, Rank rank) {
+ if (rank.equals(Rank.SUBVARIETY()) && nameStr.matches(".* var\\. .* subvar\\. .*")) {
+ nameStr = nameStr.substring(0, nameStr.indexOf(" var.")) + nameStr.substring(nameStr.indexOf(" subvar."));
+ }
+ return nameStr;
+ }
+
private void adaptRefTypeForGeneric(IBotanicalName referencedName, String refTypeStr) {
INomenclaturalReference ref = referencedName.getNomenclaturalReference();
if (ref == null){
}
}
- protected IdentifiableSource makeOriginalSource(MexicoEfloraImportState state) {
+ protected IdentifiableSource makeOriginalSource(@SuppressWarnings("unused") MexicoEfloraImportState state) {
return IdentifiableSource.NewDataImportInstance(null, null, sourceReference);
}
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, MexicoEfloraImportState state) {
String nameSpace;
- Set<String> idSet;
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
try{
handleForeignKey(rs, referenceIdSet, "IdBibliografiaSec");
}
- //reference map
- nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE;
- idSet = referenceIdSet;
- Map<String, Reference> referenceMap = getCommonService().getSourcedObjectsByIdInSourceC(Reference.class, idSet, nameSpace);
- result.put(nameSpace, referenceMap);
+ //reference map
+ nameSpace = MexicoEfloraReferenceImportBase.NAMESPACE;
+ Map<UUID,String> referenceUuidMap = new HashMap<>();
+ referenceIdSet.stream().forEach(rId->referenceUuidMap.put(state.getReferenceUuidMap().get(Integer.valueOf(rId)), rId));
+ List<Reference> references = getReferenceService().find(referenceUuidMap.keySet());
+ Map<String, Reference> referenceMap = new HashMap<>();
+ references.stream().forEach(r->referenceMap.put(referenceUuidMap.get(r.getUuid()), r));
+ result.put(nameSpace, referenceMap);
} catch (SQLException e) {
throw new RuntimeException(e);
String sql = " SELECT IdCAT "
+ " FROM " + dbTableName + " t "
+ " LEFT JOIN cv1_Controlled_vocabulary_for_name_Ranks r ON t.CategoriaTaxonomica = r.NombreCategoriaTaxonomica "
- + " WHERE t.IdCAT_AscendenteHerarquico4CDM NOT IN ('2PLANT','79217TRACH') "
+ + " WHERE t.IdCAT_AscendenteHerarquico4CDM NOT IN ('2PLANT') "
+ " ORDER BY r.Nivel1, IdCAT ";
return sql;
}
TaxonBase<?> taxonBase = taxonMap.get(uuid.toString());
try {
- if (taxonBase.isInstanceOf(Synonym.class) && accUuidStr != null) {
+ if (taxonBase == null) {
+ logger.warn(taxonId + ": Taxon does not exist");
+ continue;
+ }else if (taxonBase.isInstanceOf(Synonym.class) && accUuidStr != null) {
Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
TaxonBase<?> related = taxonMap.get(accUuidStr);
if (!related.isInstanceOf(Taxon.class)){
logger.warn(taxonId + ": Accepted taxon for synonym is not accepted: " + accUuidStr);
}else {
Taxon acc = CdmBase.deproxy(related, Taxon.class);
- //TODO type
acc.addSynonym(syn, SynonymType.SYNONYM_OF());
}
}else if (taxonBase.isInstanceOf(Taxon.class) && parentUuidStr != null) {
logger.warn(taxonId + ": Parent is not accepted: " + parentUuidStr);
}else {
Taxon parent = CdmBase.deproxy(parentBase, Taxon.class);
- //TODO
Reference parentChildReference = null;
getClassification(state).addParentChild(parent, child, parentChildReference, null);
}
TaxonName name = taxonBase.getName();
TaxonBase<?> basionymTaxon = taxonMap.get(basUuidStr);
name.addBasionym(basionymTaxon.getName());
- //TODO synrel type
}
partitioner.startDoSave();
taxaToSave.add(taxonBase);
} catch (Exception e) {
logger.warn("An exception (" +e.getMessage()+") occurred when trying to create relation for id " + taxonId + ". Relation could not be saved.");
+ e.printStackTrace();
success = false;
}
}
BGBM BDI Projects / cdmlib-apps.git / commitdiff