From 58313611b3679c2958c4dd5a0bf55220be5ceacc Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 29 Mar 2017 17:20:23 +0200 Subject: [PATCH] ref #6286 latest changes to FloraHellenica import --- .../app/greece/FloraHellenicaActivator.java | 51 +--- .../FloraHellenicaExcludedTaxonImport.java | 6 +- .../io/greece/FloraHellenicaImportBase.java | 2 - .../FloraHellenicaImportConfigurator.java | 1 + .../io/greece/FloraHellenicaTaxonImport.java | 288 +++++++++++++++--- .../io/greece/FloraHellenicaTransformer.java | 48 ++- 6 files changed, 298 insertions(+), 98 deletions(-) diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/app/greece/FloraHellenicaActivator.java b/app-import/src/main/java/eu/etaxonomy/cdm/app/greece/FloraHellenicaActivator.java index 86cd6e42..dea67238 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/app/greece/FloraHellenicaActivator.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/app/greece/FloraHellenicaActivator.java @@ -13,17 +13,12 @@ import java.util.UUID; import org.apache.log4j.Logger; -import eu.etaxonomy.cdm.api.service.ITermService; import eu.etaxonomy.cdm.app.common.CdmDestinations; import eu.etaxonomy.cdm.database.DbSchemaValidation; import eu.etaxonomy.cdm.database.ICdmDataSource; import eu.etaxonomy.cdm.io.common.CdmDefaultImport; import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK; import eu.etaxonomy.cdm.io.greece.FloraHellenicaImportConfigurator; -import eu.etaxonomy.cdm.io.greece.FloraHellenicaTransformer; -import eu.etaxonomy.cdm.model.description.Feature; -import eu.etaxonomy.cdm.model.description.FeatureNode; -import eu.etaxonomy.cdm.model.description.FeatureTree; /** * @@ -41,8 +36,8 @@ public class FloraHellenicaActivator { //database validation status (create, update, validate ...) static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE; - static final ICdmDataSource cdmDestination = CdmDestinations.localH2(); -// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql_test(); +// static final ICdmDataSource cdmDestination = CdmDestinations.localH2(); + static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql_test(); // static final ICdmDataSource cdmDestination = CdmDestinations.cdm_greece_checklist_production(); //feature tree uuid @@ -73,48 +68,16 @@ public class FloraHellenicaActivator { config.setDbSchemaValidation(hbm2dll); // config.setSourceReferenceTitle(sourceReferenceTitle); // config.setDoVocabularies(doVocabularies); + config.setUuidFeatureTree(featureTreeUuid); + config.setFeatureTreeTitle(featureTreeTitle); - CdmDefaultImport myImport = new CdmDefaultImport(); + CdmDefaultImport myImport = new CdmDefaultImport<>(); myImport.invoke(config); - FeatureTree tree = makeFeatureNodes(myImport.getCdmAppController().getTermService()); - myImport.getCdmAppController().getFeatureTreeService().saveOrUpdate(tree); +// FeatureTree tree = makeFeatureNodes(myImport.getCdmAppController().getTermService()); +// myImport.getCdmAppController().getFeatureTreeService().saveOrUpdate(tree); } - private FeatureTree makeFeatureNodes(ITermService service){ - - FeatureTree result = FeatureTree.NewInstance(featureTreeUuid); - result.setTitleCache(featureTreeTitle, true); - FeatureNode root = result.getRoot(); - FeatureNode newNode; - - Feature newFeature = (Feature)service.find(Feature.DISTRIBUTION().getUuid()); - newNode = FeatureNode.NewInstance(newFeature); - root.addChild(newNode); - - newFeature = (Feature)service.find(FloraHellenicaTransformer.uuidFloraHellenicaChorologyFeature); - newNode = FeatureNode.NewInstance(newFeature); - root.addChild(newNode); - - newFeature = (Feature)service.find(Feature.LIFEFORM().getUuid()); - newNode = FeatureNode.NewInstance(newFeature); - root.addChild(newNode); - - newFeature = (Feature)service.find(Feature.HABITAT().getUuid()); - newNode = FeatureNode.NewInstance(newFeature); - root.addChild(newNode); - - - newFeature = (Feature)service.find(Feature.NOTES().getUuid()); - newNode = FeatureNode.NewInstance(newFeature); - root.addChild(newNode); - - return result; - } - - private URI greekChecklist_OldVersion(){ - return URI.create("file:////BGBM-PESIHPC/Greece/VPG_FINAL_June_2016.xlsx"); - } private URI greekChecklist(){ return URI.create("file:////BGBM-PESIHPC/Greece/VPG_FINAL_WITH_SYNONYMS_21.01.2017.xlsx"); diff --git a/app-import/src/main/java/eu/etaxonomy/cdm/io/greece/FloraHellenicaExcludedTaxonImport.java b/app-import/src/main/java/eu/etaxonomy/cdm/io/greece/FloraHellenicaExcludedTaxonImport.java index dd9d14d9..38449eed 100644 --- a/app-import/src/main/java/eu/etaxonomy/cdm/io/greece/FloraHellenicaExcludedTaxonImport.java +++ b/app-import/src/main/java/eu/etaxonomy/cdm/io/greece/FloraHellenicaExcludedTaxonImport.java @@ -111,7 +111,7 @@ public class FloraHellenicaExcludedTaxonImport habitatVoc; private Map lifeformMap = new HashMap<>(); + private OrderedTermVocabulary statusVoc; + private PresenceAbsenceTerm rangeRestricted; + private PresenceAbsenceTerm doubtfullyRangeRestricted; + + private OrderedTermVocabulary chorologicalVoc; + private Map chorologyMap = new HashMap<>(); private static List expectedKeys= Arrays.asList(new String[]{ @@ -80,7 +89,6 @@ public class FloraHellenicaTaxonImport record = state.getOriginalRecord(); @@ -111,7 +121,6 @@ public class FloraHellenicaTaxonImport state) { + if (hasFeatureTree ){ + return; + } + if (getFeatureTreeService().find(state.getConfig().getUuidFeatureTree()) != null){ + hasFeatureTree = true; + return; + } + FeatureTree result = FeatureTree.NewInstance(state.getConfig().getUuidFeatureTree()); + result.setTitleCache(state.getConfig().getFeatureTreeTitle(), true); + FeatureNode root = result.getRoot(); + FeatureNode newNode; + + ITermService service = getTermService(); + Feature newFeature = (Feature)service.find(Feature.DISTRIBUTION().getUuid()); + newNode = FeatureNode.NewInstance(newFeature); + root.addChild(newNode); + + newFeature = (Feature)service.find(FloraHellenicaTransformer.uuidFloraHellenicaChorologyFeature); + newNode = FeatureNode.NewInstance(newFeature); + root.addChild(newNode); + + newFeature = (Feature)service.find(Feature.LIFEFORM().getUuid()); + newNode = FeatureNode.NewInstance(newFeature); + root.addChild(newNode); + + newFeature = (Feature)service.find(Feature.HABITAT().getUuid()); + newNode = FeatureNode.NewInstance(newFeature); + root.addChild(newNode); + + + newFeature = (Feature)service.find(Feature.NOTES().getUuid()); + newNode = FeatureNode.NewInstance(newFeature); + root.addChild(newNode); + + getFeatureTreeService().saveOrUpdate(result); + hasFeatureTree = true; + + } + /** * @param state * @param line @@ -139,11 +192,66 @@ public class FloraHellenicaTaxonImport record = state.getOriginalRecord(); - String value = getValue(record, CHOROLOGICAL_CATEGOGY); - //TODO + String valueStr = getValue(record, CHOROLOGICAL_CATEGOGY); + String value = valueStr; + if (value == null){ + return; + } + Feature choroFeature = getFeature(state, FloraHellenicaTransformer.uuidFloraHellenicaChorologyFeature, + "Chorology", "The Chorological Category", "Choro", null); + CategoricalData catData = CategoricalData.NewInstance(choroFeature); + + String[] splits = value.split(" & "); + replaceDirection(splits, line); + for (String split: splits){ + String[] splitsA = split.split("/"); + for (String splitA : splitsA){ + String[] splitsB = splitA.split(", "); + for (String splitB : splitsB){ + splitB = normalizeChorology(splitB); + State choroTerm = chorologyMap.get(splitB); + if (choroTerm == null){ + logger.warn(line + "Some chorology could not be recognized in: " + value + "; Term was: " +splitB); + }else{ + catData.addStateData(choroTerm); + } + } + } + } + desc.addElement(catData); + } + /** + * @param splitB + * @return + */ + private String normalizeChorology(String choroStr) { + choroStr = choroStr.trim() + .replace("BK", "Bk") + .replace("Austral.", "Austr.") + .replace("trop.As.", "trop. As."); + if (choroStr.startsWith("[") && !choroStr.endsWith("]")){ + choroStr += "]"; + }else if (!choroStr.startsWith("[") && choroStr.endsWith("]")){ + choroStr = "[" + choroStr; + } + return choroStr; + } + /** + * @param splits + * @param line + */ + private void replaceDirection(String[] splits, String line) { + if (splits.length > 1){ + String[] divs = splits[1].split("-"); + if (divs.length == 2){ + splits[0] = splits[0] + "-" + divs[1]; + }else{ + logger.warn(line + "Splits[1] has not expected format: " + splits[1]); + } + } } /** @@ -480,27 +588,27 @@ public class FloraHellenicaTaxonImport record = state.getOriginalRecord(); String value = getValue(record, key); - if (value != null){ - NamedArea area = getNamedArea(state, uuid, null, null, null, null, null); - Distribution dist; + NamedArea area = getNamedArea(state, uuid, null, null, null, null, null); + Distribution dist; + if (value == null || ".".equals(value) ){ + dist = Distribution.NewInstance(area, PresenceAbsenceTerm.NATIVE()); if (".".equals(value)){ - dist = Distribution.NewInstance(area, PresenceAbsenceTerm.ABSENT()); logger.warn(line + "'.' Should not exist anmore as a distribution status: '" + value + "' for " + key); - }else if ("Range-restricted".equals(value)){ - dist = Distribution.NewInstance(area, PresenceAbsenceTerm.ENDEMIC_FOR_THE_RELEVANT_AREA()); - }else if ("?Range-restricted".equals(value)){ - dist = Distribution.NewInstance(area, PresenceAbsenceTerm.ENDEMIC_FOR_THE_RELEVANT_AREA()); - }else if ("Xenophyte".equals(value)){ - dist = Distribution.NewInstance(area, PresenceAbsenceTerm.INTRODUCED()); - }else if ("?Xenophyte".equals(value)){ - dist = Distribution.NewInstance(area, PresenceAbsenceTerm.INTRODUCED_DOUBTFULLY_INTRODUCED()); - }else { - logger.warn(line + "Not matching status. This should not happpen '" + value + "' for " + key); - return; } - desc.addElement(dist); - dist.addImportSource(id, getWorksheetName(), getSourceCitation(state), line); + }else if ("Range-restricted".equals(value)){ + dist = Distribution.NewInstance(area, rangeRestricted); + }else if ("?Range-restricted".equals(value)){ + dist = Distribution.NewInstance(area, doubtfullyRangeRestricted); + }else if ("Xenophyte".equals(value)){ + dist = Distribution.NewInstance(area, PresenceAbsenceTerm.INTRODUCED()); + }else if ("?Xenophyte".equals(value)){ + dist = Distribution.NewInstance(area, PresenceAbsenceTerm.INTRODUCED_DOUBTFULLY_INTRODUCED()); + }else { + logger.warn(line + "Not matching status. This should not happpen '" + value + "' for " + key); + return; } + desc.addElement(dist); + dist.addImportSource(id, getWorksheetName(), getSourceCitation(state), line); } @SuppressWarnings("unchecked") @@ -577,27 +685,71 @@ public class FloraHellenicaTaxonImport state, String abbrevLabel, String label, - String string3, UUID uuidChorological) { - State chorological = addState(state, abbrevLabel, label, label, uuidChorological, chorologicalVoc); - - + String desc, UUID uuidChorological) { + desc = isBlank(desc)? label : desc; + State chorologyTerm = addState(state, abbrevLabel, label, desc, uuidChorological, chorologicalVoc); + chorologyMap.put(abbrevLabel, chorologyTerm); } /** @@ -635,11 +787,21 @@ public class FloraHellenicaTaxonImport state, + String abbrev, String stateLabel, String description, UUID uuid, OrderedTermVocabulary voc) { + PresenceAbsenceTerm newStatus = PresenceAbsenceTerm.NewPresenceInstance( + description, stateLabel, abbrev); + newStatus.setUuid(uuid); + newStatus.setIdInVocabulary(abbrev); + newStatus.setSymbol(abbrev); + voc.addTerm(newStatus); + return newStatus; + } + private void initHabitatVocabulary(SimpleExcelTaxonImportState state) { if (habitatVoc == null){ UUID uuid = FloraHellenicaTransformer.uuidFloraHellenicaHabitatVoc; @@ -650,6 +812,35 @@ public class FloraHellenicaTaxonImport state) { + if (statusVoc == null){ + UUID uuid = FloraHellenicaTransformer.uuidFloraHellenicaStatusVoc; + statusVoc = (OrderedTermVocabulary)this.getVocabularyService().find(uuid); + if (statusVoc == null){ + createStatusVoc(state, uuid); + } + } + } + + /** + * @param state + */ + private void createStatusVoc(SimpleExcelTaxonImportState state, UUID vocUuid) { + //voc + URI termSourceUri = null; + String label = "Checklist of Greece Status"; + String description = "Status as used in the Checklist of Greece"; + statusVoc = OrderedTermVocabulary.NewInstance(TermType.PresenceAbsenceTerm, + description, label, null, termSourceUri); + statusVoc.setUuid(vocUuid); + + rangeRestricted = addStatus(state, "RR", "Range-restricted", "", FloraHellenicaTransformer.uuidStatusRangeRestricted, statusVoc); + doubtfullyRangeRestricted = addStatus(state, "?RR", "?Range-restricted", "", FloraHellenicaTransformer.uuidStatusRangeRestrictedDoubtfully, statusVoc); + + this.getVocabularyService().save(statusVoc); + return; + } + /** * @param state @@ -703,9 +894,10 @@ public class FloraHellenicaTaxonImport