--- /dev/null
+/**
+* Copyright (C) 2007 EDIT
+* European Distributed Institute of Taxonomy
+* http://www.e-taxonomy.eu
+*
+* The contents of this file are subject to the Mozilla Public License Version 1.1
+* See LICENSE.TXT at the top of this package for the full license terms.
+*/
+
+package eu.etaxonomy.cdm.app.iapt;
+
+import java.io.File;
+import java.net.URI;
+import java.util.UUID;
+
+import org.apache.log4j.Logger;
+
+import eu.etaxonomy.cdm.app.common.CdmDestinations;
+import eu.etaxonomy.cdm.database.DbSchemaValidation;
+import eu.etaxonomy.cdm.database.ICdmDataSource;
+import eu.etaxonomy.cdm.io.common.CdmDefaultImport;
+import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
+import eu.etaxonomy.cdm.io.iapt.IAPTImportConfigurator;
+import eu.etaxonomy.cdm.model.reference.Reference;
+import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
+
+
+/**
+ * @author a.kohlbecker
+ * @date Jul 26, 2016
+ *
+ */
+public class IAPTActivator {
+ private static final Logger logger = Logger.getLogger(IAPTActivator.class);
+
+ //database validation status (create, update, validate ...)
+ static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE;
+
+ static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
+// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql_test();
+// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_cuba_production();
+
+ static boolean invers = true;
+
+ static boolean include = !invers;
+
+ //classification
+ static final UUID classificationUuid = UUID.fromString("8c51efb4-3d67-4bea-8f87-4bc1cba1310d");
+ private static final String classificationName = "IAPT";
+ static final String sourceReferenceTitle = "IAPT Import";
+
+ //check - import
+ static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
+
+ static final boolean doTaxa = false;
+
+
+ private void doImport(ICdmDataSource cdmDestination){
+
+ URI source = iapt(); //just any
+
+ Reference secRef = ReferenceFactory.newDatabase();
+ secRef.setTitle("IAPT");
+
+ //make Source
+ IAPTImportConfigurator config= IAPTImportConfigurator.NewInstance(source, cdmDestination);
+ config.setClassificationUuid(classificationUuid);
+ config.setClassificationName(classificationName);
+ config.setCheck(check);
+ config.setDbSchemaValidation(hbm2dll);
+ config.setSourceReferenceTitle(sourceReferenceTitle);
+ config.setSecReference(secRef);
+
+ CdmDefaultImport<IAPTImportConfigurator> myImport = new CdmDefaultImport<>();
+
+ doSingleSource(iapt(), config, myImport);
+
+ System.exit(0);
+
+ }
+
+ /**
+ * @param source
+ * @param config
+ * @param myImport
+ */
+ private void doSingleSource(URI source, IAPTImportConfigurator config, CdmDefaultImport<IAPTImportConfigurator> myImport) {
+ config.setSource(source);
+ String fileName = source.toString();
+ fileName = fileName.substring(fileName.lastIndexOf("/") + 1 );
+
+ String message = "Start import from ("+ fileName + ") ...";
+ System.out.println(message);
+ logger.warn(message);
+ config.setSourceReference(getSourceReference(fileName));
+ myImport.invoke(config);
+
+ System.out.println("End import from ("+ source.toString() + ")...");
+ }
+
+ private final Reference inRef = ReferenceFactory.newGeneric();
+ private Reference getSourceReference(String string) {
+ Reference result = ReferenceFactory.newGeneric();
+ result.setTitleCache(string, true);
+ result.setInReference(inRef);
+ inRef.setTitleCache(sourceReferenceTitle, true);
+ return result;
+ }
+
+
+
+ public static URI iapt() {
+ File f = new File("~/data/Projekte/Algea Name Registry/registry/sources/IAPT/Registration_DB_from_BGBM17-cleaned.xls");
+ return f.toURI();
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ IAPTActivator me = new IAPTActivator();
+ me.doImport(cdmDestination);
+ }
+
+}
--- /dev/null
+/**
+ * Copyright (C) 2007 EDIT
+ * European Distributed Institute of Taxonomy
+ * http://www.e-taxonomy.eu
+ *
+ * The contents of this file are subject to the Mozilla Public License Version 1.1
+ * See LICENSE.TXT at the top of this package for the full license terms.
+ */
+
+package eu.etaxonomy.cdm.io.iapt;
+
+import eu.etaxonomy.cdm.common.CdmUtils;
+import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImport;
+import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;
+import eu.etaxonomy.cdm.model.common.*;
+import eu.etaxonomy.cdm.model.name.*;
+import eu.etaxonomy.cdm.model.reference.INomenclaturalReference;
+import eu.etaxonomy.cdm.model.reference.Reference;
+import eu.etaxonomy.cdm.model.taxon.*;
+import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
+import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
+import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+/**
+ * @author a.mueller
+ * @created 05.01.2016
+ */
+
+@Component
+public class IAPTExcelImport<CONFIG extends IAPTImportConfigurator> extends SimpleExcelTaxonImport<CONFIG> {
+ private static final long serialVersionUID = -747486709409732371L;
+ private static final Logger logger = Logger.getLogger(IAPTExcelImport.class);
+
+
+ private static UUID ROOT_UUID = UUID.fromString("4137fd2a-20f6-4e70-80b9-f296daf51d82");
+
+ private static INonViralNameParser<?> nameParser = NonViralNameParserImpl.NewInstance();
+
+ private final static String REGISTRATIONNO_PK= "RegistrationNo_Pk";
+ private final static String HIGHERTAXON= "HigherTaxon";
+ private final static String FULLNAME= "FullName";
+ private final static String AUTHORSSPELLING= "AuthorsSpelling";
+ private final static String LITSTRING= "LitString";
+ private final static String REGISTRATION= "Registration";
+ private final static String TYPE= "Type";
+ private final static String CAVEATS= "Caveats";
+ private final static String FULLBASIONYM= "FullBasionym";
+ private final static String FULLSYNSUBST= "FullSynSubst";
+ private final static String NOTESTXT= "NotesTxt";
+ private final static String REGDATE= "RegDate";
+ private final static String NAMESTRING= "NameString";
+ private final static String BASIONYMSTRING= "BasionymString";
+ private final static String SYNSUBSTSTR= "SynSubstStr";
+ private final static String AUTHORSTRING= "AuthorString";
+
+ private static List<String> expectedKeys= Arrays.asList(new String[]{
+ REGISTRATIONNO_PK, HIGHERTAXON, FULLNAME, AUTHORSSPELLING, LITSTRING, REGISTRATION, TYPE, CAVEATS, FULLBASIONYM, FULLSYNSUBST, NOTESTXT, REGDATE, NAMESTRING, BASIONYMSTRING, SYNSUBSTSTR, AUTHORSTRING});
+
+
+ private Taxon makeTaxon(HashMap<String, String> record, SimpleExcelTaxonImportState<CONFIG> state,
+ TaxonNode higherTaxonNode, boolean isSynonym) {
+
+ String line = state.getCurrentLine() + ": ";
+
+ String fullNameStr = getValue(record, FULLNAME);
+ String nameStr = getValue(record, NAMESTRING);
+ String authorStr = getValue(record, AUTHORSTRING);
+
+ String sourceReference = getValue(record, LITSTRING);
+
+ BotanicalName taxonName = (BotanicalName) nameParser.parseFullName(fullNameStr, NomenclaturalCode.ICNAFP, null);
+ if (taxonName.isProtectedTitleCache()) {
+ logger.warn(line + "Name could not be parsed: " + fullNameStr);
+ } else {
+ // Check Name
+ if (!taxonName.getNameCache().equals(nameStr)) {
+ logger.warn(line + "parsed nameCache differs from " + NAMESTRING + " : " + taxonName.getNameCache() + " <> " + nameStr);
+ }
+ // Check Author
+ INomenclaturalReference nomRef = taxonName.getNomenclaturalReference();
+ if (!nomRef.getAuthorship().getTitleCache().equals(authorStr)) {
+ logger.warn(line + "parsed nomRef.authorship differs from " + AUTHORSTRING + " : " + nomRef.getAuthorship().getTitleCache() + " <> " + authorStr);
+ // preserve current titleCache
+ taxonName.setProtectedTitleCache(true);
+ try {
+ nameParser.parseAuthors(taxonName, authorStr);
+ } catch (StringNotParsableException e) {
+ logger.error(" " + authorStr + " can not be parsed");
+ }
+ }
+
+ // deduplicate
+ replaceAuthorNamesAndNomRef(state, taxonName);
+ }
+
+ Reference sec = state.getConfig().getSecReference();
+ Taxon taxon = Taxon.NewInstance(taxonName, sec);
+ getTaxonService().save(taxon);
+ if(higherTaxonNode != null){
+ higherTaxonNode.addChildTaxon(taxon, null, null);
+ getTaxonNodeService().save(higherTaxonNode);
+ }
+
+ return taxon;
+
+ }
+
+ /**
+ * @param state
+ * @return
+ */
+ private TaxonNode getClassification(IAPTImportState state) {
+
+ Classification classification = state.getClassification();
+ if (classification == null){
+ IAPTImportConfigurator config = state.getConfig();
+ classification = Classification.NewInstance(state.getConfig().getClassificationName());
+ classification.setUuid(config.getClassificationUuid());
+ classification.setReference(config.getSecReference());
+ classification = getClassificationService().find(state.getConfig().getClassificationUuid());
+ }
+ TaxonNode rootNode = state.getRootNode();
+ if (rootNode == null){
+ rootNode = getTaxonNodeService().find(ROOT_UUID);
+ }
+ if (rootNode == null){
+ Reference sec = state.getSecReference();
+ if (classification == null){
+ String classificationName = state.getConfig().getClassificationName();
+ //TODO
+ Language language = Language.DEFAULT();
+ classification = Classification.NewInstance(classificationName, sec, language);
+ state.setClassification(classification);
+ classification.setUuid(state.getConfig().getClassificationUuid());
+ classification.getRootNode().setUuid(ROOT_UUID);
+ }
+
+ getClassificationService().save(classification);
+ rootNode = classification.getRootNode();
+ }
+ return rootNode;
+ }
+
+
+ /**
+ * @param record
+ * @param originalKey
+ * @return
+ */
+ private String getValue(HashMap<String, String> record, String originalKey) {
+ String value = record.get(originalKey);
+ if (! StringUtils.isBlank(value)) {
+ if (logger.isDebugEnabled()) { logger.debug(originalKey + ": " + value); }
+ value = CdmUtils.removeDuplicateWhitespace(value.trim()).toString();
+ return value;
+ }else{
+ return null;
+ }
+ }
+
+
+
+ /**
+ * Stores taxa records in DB
+ */
+ @Override
+ protected void firstPass(SimpleExcelTaxonImportState<CONFIG> state) {
+
+ boolean isSynonymOnly = false;
+
+ String line = state.getCurrentLine() + ": ";
+ HashMap<String, String> record = state.getOriginalRecord();
+
+ Set<String> keys = record.keySet();
+ for (String key: keys) {
+ if (! expectedKeys.contains(key)){
+ logger.warn(line + "Unexpected Key: " + key);
+ }
+ }
+
+ //higherTaxon
+ TaxonNode higherTaxon = getHigherTaxon(record, (IAPTImportState)state);
+
+ //Taxon
+ Taxon taxon = makeTaxon(record, state, higherTaxon, isSynonymOnly);
+ if (taxon == null && ! isSynonymOnly){
+ logger.warn(line + "taxon could not be created and is null");
+ return;
+ }
+ ((IAPTImportState)state).setCurrentTaxon(taxon);
+
+ //(Notas)
+ //makeNotes(record, state);
+
+ //Syn.
+ //makeSynonyms(record, state, !isSynonymOnly);
+
+
+ return;
+ }
+
+ private TaxonNode getHigherTaxon(HashMap<String, String> record, IAPTImportState state) {
+ String higherTaxaString = record.get(HIGHERTAXON);
+ // higherTaxaString is like
+ // - DICOTYLEDONES: LEGUMINOSAE: MIMOSOIDEAE
+ // - FOSSIL DICOTYLEDONES: PROTEACEAE
+ // - [fungi]
+ // - [no group assigned]
+ if(higherTaxaString.equals("[no group assigned]")){
+ return null;
+ }
+ String[] higherTaxaNames = higherTaxaString.toLowerCase().replaceAll("[\\[\\]]", "").split(":");
+ TaxonNode higherTaxonNode = null;
+
+ ITaxonTreeNode rootNode = getClassification(state);
+ for (String htn : higherTaxaNames) {
+ htn = htn.trim();
+ Taxon higherTaxon = state.getHigherTaxon(htn);
+ if (higherTaxon != null){
+ higherTaxonNode = higherTaxon.getTaxonNodes().iterator().next();
+ }else{
+ BotanicalName name = makeHigherTaxonName(state, htn);
+ Reference sec = state.getSecReference();
+ higherTaxon = Taxon.NewInstance(name, sec);
+ higherTaxonNode = rootNode.addChildTaxon(higherTaxon, sec, null);
+ state.putHigherTaxon(htn, higherTaxon);
+ rootNode = higherTaxonNode;
+ }
+ }
+ return higherTaxonNode;
+ }
+
+ private BotanicalName makeHigherTaxonName(IAPTImportState state, String name) {
+ // Abteilung: -phyta (bei Pflanzen), -mycota (bei Pilzen)
+ // Unterabteilung: -phytina (bei Pflanzen), -mycotina (bei Pilzen)
+ // Klasse: -opsida (bei Pflanzen), -phyceae (bei Algen), -mycetes (bei Pilzen)
+ // Unterklasse: -idae (bei Pflanzen), -phycidae (bei Algen), -mycetidae (bei Pilzen)
+ // Ordnung: -ales
+ // Unterordnung: -ineae
+ // Familie: -aceae
+ // Unterfamilie: -oideae
+ // Tribus: -eae
+ // Subtribus: -inae
+ Rank rank = Rank.UNKNOWN_RANK();
+ if(name.matches("phyta$|mycota$")){
+ rank = Rank.SECTION_BOTANY();
+ } else if(name.matches("phytina$|mycotina$")){
+ rank = Rank.SUBSECTION_BOTANY();
+ } else if(name.matches("opsida$|phyceae$|mycetes$")){
+ rank = Rank.CLASS();
+ } else if(name.matches("idae$|phycidae$|mycetidae$")){
+ rank = Rank.SUBCLASS();
+ } else if(name.matches("ales$")){
+ rank = Rank.ORDER();
+ } else if(name.matches("ineae$")){
+ rank = Rank.SUBORDER();
+ } else if(name.matches("aceae$")){
+ rank = Rank.FAMILY();
+ } else if(name.matches("oideae$")){
+ rank = Rank.SUBFAMILY();
+ } else if(name.matches("eae$")){
+ rank = Rank.TRIBE();
+ } else if(name.matches("inae$")){
+ rank = Rank.SUBTRIBE();
+ }
+
+ BotanicalName taxonName = BotanicalName.NewInstance(rank);
+ taxonName.addSource(makeOriginalSource(state));
+ taxonName.setGenusOrUninomial(StringUtils.capitalize(name));
+ return taxonName;
+ }
+
+
+ /**
+ * @param state
+ * @return
+ */
+ private IdentifiableSource makeOriginalSource(IAPTImportState state) {
+ return IdentifiableSource.NewDataImportInstance("line: " + state.getCurrentLine(), null, state.getConfig().getSourceReference());
+ }
+
+
+ private Reference makeReference(IAPTImportState state, UUID uuidRef) {
+ Reference ref = state.getReference(uuidRef);
+ if (ref == null){
+ ref = getReferenceService().find(uuidRef);
+ state.putReference(uuidRef, ref);
+ }
+ return ref;
+ }
+
+
+
+}
--- /dev/null
+// $Id$
+/**
+* Copyright (C) 2007 EDIT
+* European Distributed Institute of Taxonomy
+* http://www.e-taxonomy.eu
+*
+* The contents of this file are subject to the Mozilla Public License Version 1.1
+* See LICENSE.TXT at the top of this package for the full license terms.
+*/
+
+package eu.etaxonomy.cdm.io.iapt;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;
+import org.apache.log4j.Logger;
+
+import eu.etaxonomy.cdm.io.excel.common.ExcelImportState;
+import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase;
+import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
+import eu.etaxonomy.cdm.model.name.BotanicalName;
+import eu.etaxonomy.cdm.model.reference.Reference;
+import eu.etaxonomy.cdm.model.taxon.Classification;
+import eu.etaxonomy.cdm.model.taxon.Taxon;
+import eu.etaxonomy.cdm.model.taxon.TaxonNode;
+
+/**
+ * @author a.mueller
+ * @created 11.05.2009
+ */
+public class IAPTImportState extends SimpleExcelTaxonImportState<IAPTImportConfigurator> {
+ @SuppressWarnings("unused")
+ private static final Logger logger = Logger.getLogger(IAPTImportState.class);
+
+ private final Map<String, Taxon> higherTaxonTaxonMap = new HashMap<String, Taxon>();
+
+ private final Map<String, UUID> higherTaxonUuidMap = new HashMap<String, UUID>();
+
+ private final Map<String, BotanicalName> familyNameMap = new HashMap<String, BotanicalName>();
+
+ //classification
+ private Classification classification;
+ public Classification getClassification() {return classification;}
+ public void setClassification(Classification classification) {this.classification = classification;}
+
+ //current taxon
+ private Taxon currentTaxon;
+ public Taxon getCurrentTaxon() {return currentTaxon;}
+ public void setCurrentTaxon(Taxon currentTaxon) {this.currentTaxon = currentTaxon;}
+
+ //rootNode
+ private TaxonNode rootNode;
+ public void setRootNode(TaxonNode rootNode) {this.rootNode = rootNode;}
+ public TaxonNode getRootNode() { return rootNode;}
+
+ private Reference secReference;
+ public Reference getSecReference() {return secReference;}
+ public void setSecReference(Reference secReference) {this.secReference = secReference;}
+
+ private PresenceAbsenceTerm highestStatusForTaxon;
+ public PresenceAbsenceTerm getHighestStatusForTaxon(){return highestStatusForTaxon;}
+ public void setHighestStatusForTaxon(PresenceAbsenceTerm highestStatusForTaxon){this.highestStatusForTaxon = highestStatusForTaxon;}
+
+ //Constructor
+ public IAPTImportState(IAPTImportConfigurator config) {
+ super(config);
+ }
+
+ //higher taxon
+ public Taxon getHigherTaxon(String higherName) {
+ return higherTaxonTaxonMap.get(higherName);
+ }
+ public Taxon putHigherTaxon(String higherName, Taxon taxon) {
+ return higherTaxonTaxonMap.put(higherName, taxon);
+ }
+ public Taxon removeHigherTaxon(String higherName) {
+ return higherTaxonTaxonMap.remove(higherName);
+ }
+ public boolean containsHigherTaxon(String higherName) {
+ return higherTaxonTaxonMap.containsKey(higherName);
+ }
+
+ //higher taxon uuid
+ public UUID getHigherTaxonUuid(String higherName) {
+ return higherTaxonUuidMap.get(higherName);
+ }
+ public UUID putHigherTaxon(String higherName, UUID uuid) {
+ return higherTaxonUuidMap.put(higherName, uuid);
+ }
+ public UUID removeHigherTaxonUuid(String higherName) {
+ return higherTaxonUuidMap.remove(higherName);
+ }
+ public boolean containsHigherTaxonUuid(String higherName) {
+ return higherTaxonUuidMap.containsKey(higherName);
+ }
+
+ //family names
+ public BotanicalName getFamilyName(String familyStr) {
+ return familyNameMap.get(familyStr);
+ }
+ public void putFamilyName(String familyStr, BotanicalName name) {
+ familyNameMap.put(familyStr, name);
+ }
+
+
+ Map<UUID, Reference> refMap = new HashMap<UUID, Reference>();
+ //reference
+ public Reference getReference(UUID uuidRef) {
+ return refMap.get(uuidRef);
+ }
+ public void putReference(UUID uuidRef, Reference ref) {
+ refMap.put(uuidRef, ref);
+ }
+
+
+}
BGBM BDI Projects / cdmlib-apps.git / commitdiff