EDIT

/**

* Copyright (C) 2007 EDIT

* European Distributed Institute of Taxonomy

* http://www.e-taxonomy.eu

*

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.app.iapt;

import java.io.File;

import java.net.URI;

import java.util.UUID;

import org.apache.log4j.Logger;

import eu.etaxonomy.cdm.app.common.CdmDestinations;

import eu.etaxonomy.cdm.database.DbSchemaValidation;

import eu.etaxonomy.cdm.database.ICdmDataSource;

import eu.etaxonomy.cdm.io.common.CdmDefaultImport;

import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;

import eu.etaxonomy.cdm.io.iapt.IAPTImportConfigurator;

import eu.etaxonomy.cdm.model.reference.Reference;

import eu.etaxonomy.cdm.model.reference.ReferenceFactory;

/**

 * @author a.kohlbecker

 * @date Jul 26, 2016

 *

 */

public class IAPTActivator {

    private static final Logger logger = Logger.getLogger(IAPTActivator.class);

    //database validation status (create, update, validate ...)

    static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE;

    static final ICdmDataSource cdmDestination = CdmDestinations.localH2();

//  static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql_test();

//    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_cuba_production();

    static boolean invers = true;

    static boolean include = !invers;

    //classification

    static final UUID classificationUuid = UUID.fromString("8c51efb4-3d67-4bea-8f87-4bc1cba1310d");

    private static final String classificationName = "IAPT";

    static final String sourceReferenceTitle = "IAPT Import";

    //check - import

    static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;

    static final boolean doTaxa = false;

    private void doImport(ICdmDataSource cdmDestination){

        URI source = iapt();  //just any

        Reference secRef = ReferenceFactory.newDatabase();

        secRef.setTitle("IAPT");

        //make Source

        IAPTImportConfigurator config= IAPTImportConfigurator.NewInstance(source, cdmDestination);

        config.setClassificationUuid(classificationUuid);

        config.setClassificationName(classificationName);

        config.setCheck(check);

        config.setDbSchemaValidation(hbm2dll);

        config.setSourceReferenceTitle(sourceReferenceTitle);

        config.setSecReference(secRef);

        CdmDefaultImport<IAPTImportConfigurator> myImport = new CdmDefaultImport<>();

        doSingleSource(iapt(), config, myImport);

        System.exit(0);

    }

    /**

     * @param source

     * @param config

     * @param myImport

     */

    private void doSingleSource(URI source, IAPTImportConfigurator config, CdmDefaultImport<IAPTImportConfigurator> myImport) {

        config.setSource(source);

        String fileName = source.toString();

        fileName = fileName.substring(fileName.lastIndexOf("/") + 1 );

        String message = "Start import from ("+ fileName + ") ...";

        System.out.println(message);

        logger.warn(message);

        config.setSourceReference(getSourceReference(fileName));

        myImport.invoke(config);

        System.out.println("End import from ("+ source.toString() + ")...");

    }

    private final Reference inRef = ReferenceFactory.newGeneric();

    private Reference getSourceReference(String string) {

        Reference result = ReferenceFactory.newGeneric();

        result.setTitleCache(string, true);

        result.setInReference(inRef);

        inRef.setTitleCache(sourceReferenceTitle, true);

        return result;

    }

    public static URI iapt() {

        File f = new File("~/data/Projekte/Algea Name Registry/registry/sources/IAPT/Registration_DB_from_BGBM17-cleaned.xls");

        return f.toURI();

    }

    /**

     * @param args

     */

    public static void main(String[] args) {

        IAPTActivator me = new IAPTActivator();

        me.doImport(cdmDestination);

    }

}

/**

 * Copyright (C) 2007 EDIT

 * European Distributed Institute of Taxonomy

 * http://www.e-taxonomy.eu

 *

 * The contents of this file are subject to the Mozilla Public License Version 1.1

 * See LICENSE.TXT at the top of this package for the full license terms.

 */

package eu.etaxonomy.cdm.io.iapt;

import eu.etaxonomy.cdm.common.CdmUtils;

import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImport;

import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;

import eu.etaxonomy.cdm.model.common.*;

import eu.etaxonomy.cdm.model.name.*;

import eu.etaxonomy.cdm.model.reference.INomenclaturalReference;

import eu.etaxonomy.cdm.model.reference.Reference;

import eu.etaxonomy.cdm.model.taxon.*;

import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;

import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;

import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;

import org.apache.commons.lang.StringUtils;

import org.apache.log4j.Logger;

import org.springframework.stereotype.Component;

import java.util.*;

/**

 * @author a.mueller

 * @created 05.01.2016

 */

@Component

public class IAPTExcelImport<CONFIG extends IAPTImportConfigurator> extends SimpleExcelTaxonImport<CONFIG> {

    private static final long serialVersionUID = -747486709409732371L;

    private static final Logger logger = Logger.getLogger(IAPTExcelImport.class);

    private static UUID ROOT_UUID = UUID.fromString("4137fd2a-20f6-4e70-80b9-f296daf51d82");

    private static INonViralNameParser<?> nameParser = NonViralNameParserImpl.NewInstance();

    private final static String REGISTRATIONNO_PK= "RegistrationNo_Pk";

    private final static String HIGHERTAXON= "HigherTaxon";

    private final static String FULLNAME= "FullName";

    private final static String AUTHORSSPELLING= "AuthorsSpelling";

    private final static String LITSTRING= "LitString";

    private final static String REGISTRATION= "Registration";

    private final static String TYPE= "Type";

    private final static String CAVEATS= "Caveats";

    private final static String FULLBASIONYM= "FullBasionym";

    private final static String FULLSYNSUBST= "FullSynSubst";

    private final static String NOTESTXT= "NotesTxt";

    private final static String REGDATE= "RegDate";

    private final static String NAMESTRING= "NameString";

    private final static String BASIONYMSTRING= "BasionymString";

    private final static String SYNSUBSTSTR= "SynSubstStr";

    private final static String AUTHORSTRING= "AuthorString";

    private  static List<String> expectedKeys= Arrays.asList(new String[]{

            REGISTRATIONNO_PK, HIGHERTAXON, FULLNAME, AUTHORSSPELLING, LITSTRING, REGISTRATION, TYPE, CAVEATS, FULLBASIONYM, FULLSYNSUBST, NOTESTXT, REGDATE, NAMESTRING, BASIONYMSTRING, SYNSUBSTSTR, AUTHORSTRING});

    private Taxon makeTaxon(HashMap<String, String> record, SimpleExcelTaxonImportState<CONFIG> state,

                            TaxonNode higherTaxonNode, boolean isSynonym) {

        String line = state.getCurrentLine() + ": ";

        String fullNameStr = getValue(record, FULLNAME);

        String nameStr = getValue(record, NAMESTRING);

        String authorStr = getValue(record, AUTHORSTRING);

        String sourceReference = getValue(record, LITSTRING);

        BotanicalName taxonName = (BotanicalName) nameParser.parseFullName(fullNameStr, NomenclaturalCode.ICNAFP, null);

        if (taxonName.isProtectedTitleCache()) {

            logger.warn(line + "Name could not be parsed: " + fullNameStr);

        } else {

            // Check Name

            if (!taxonName.getNameCache().equals(nameStr)) {

                logger.warn(line + "parsed nameCache differs from " + NAMESTRING + " : " + taxonName.getNameCache() + " <> " + nameStr);

            }

            // Check Author

            INomenclaturalReference nomRef = taxonName.getNomenclaturalReference();

            if (!nomRef.getAuthorship().getTitleCache().equals(authorStr)) {

                logger.warn(line + "parsed nomRef.authorship differs from " + AUTHORSTRING + " : " + nomRef.getAuthorship().getTitleCache() + " <> " + authorStr);

                // preserve current titleCache

                taxonName.setProtectedTitleCache(true);

                try {

                    nameParser.parseAuthors(taxonName, authorStr);

                } catch (StringNotParsableException e) {

                    logger.error("    " + authorStr + " can not be parsed");

                }

            }

            // deduplicate

            replaceAuthorNamesAndNomRef(state, taxonName);

        }

        Reference sec = state.getConfig().getSecReference();

        Taxon taxon = Taxon.NewInstance(taxonName, sec);

        getTaxonService().save(taxon);

        if(higherTaxonNode != null){

            higherTaxonNode.addChildTaxon(taxon, null, null);

            getTaxonNodeService().save(higherTaxonNode);

        }

        return taxon;

    }

    /**

     * @param state

     * @return

     */

    private TaxonNode getClassification(IAPTImportState state) {

        Classification classification = state.getClassification();

        if (classification == null){

            IAPTImportConfigurator config = state.getConfig();

            classification = Classification.NewInstance(state.getConfig().getClassificationName());

            classification.setUuid(config.getClassificationUuid());

            classification.setReference(config.getSecReference());

            classification = getClassificationService().find(state.getConfig().getClassificationUuid());

        }

        TaxonNode rootNode = state.getRootNode();

        if (rootNode == null){

            rootNode = getTaxonNodeService().find(ROOT_UUID);

        }

        if (rootNode == null){

            Reference sec = state.getSecReference();

            if (classification == null){

                String classificationName = state.getConfig().getClassificationName();

                //TODO

                Language language = Language.DEFAULT();

                classification = Classification.NewInstance(classificationName, sec, language);

                state.setClassification(classification);

                classification.setUuid(state.getConfig().getClassificationUuid());

                classification.getRootNode().setUuid(ROOT_UUID);

            }

            getClassificationService().save(classification);

            rootNode = classification.getRootNode();

        }

        return rootNode;

    }

    /**

     * @param record

     * @param originalKey

     * @return

     */

    private String getValue(HashMap<String, String> record, String originalKey) {

        String value = record.get(originalKey);

        if (! StringUtils.isBlank(value)) {

        	if (logger.isDebugEnabled()) { logger.debug(originalKey + ": " + value); }

        	value = CdmUtils.removeDuplicateWhitespace(value.trim()).toString();

        	return value;

        }else{

        	return null;

        }

    }

	/**

	 *  Stores taxa records in DB

	 */

	@Override

    protected void firstPass(SimpleExcelTaxonImportState<CONFIG> state) {

	    boolean isSynonymOnly = false;

        String line = state.getCurrentLine() + ": ";

        HashMap<String, String> record = state.getOriginalRecord();

        Set<String> keys = record.keySet();

        for (String key: keys) {

            if (! expectedKeys.contains(key)){

                logger.warn(line + "Unexpected Key: " + key);

            }

        }

        //higherTaxon

        TaxonNode higherTaxon = getHigherTaxon(record, (IAPTImportState)state);

       //Taxon

        Taxon taxon = makeTaxon(record, state, higherTaxon, isSynonymOnly);

        if (taxon == null && ! isSynonymOnly){

            logger.warn(line + "taxon could not be created and is null");

            return;

        }

        ((IAPTImportState)state).setCurrentTaxon(taxon);

        //(Notas)

        //makeNotes(record, state);

        //Syn.

        //makeSynonyms(record, state, !isSynonymOnly);

		return;

    }

    private TaxonNode getHigherTaxon(HashMap<String, String> record, IAPTImportState state) {

        String higherTaxaString = record.get(HIGHERTAXON);

        // higherTaxaString is like

        // - DICOTYLEDONES: LEGUMINOSAE: MIMOSOIDEAE

        // - FOSSIL DICOTYLEDONES: PROTEACEAE

        // - [fungi]

        // - [no group assigned]

        if(higherTaxaString.equals("[no group assigned]")){

            return null;

        }

        String[] higherTaxaNames = higherTaxaString.toLowerCase().replaceAll("[\\[\\]]", "").split(":");

        TaxonNode higherTaxonNode = null;

        ITaxonTreeNode rootNode = getClassification(state);

        for (String htn :  higherTaxaNames) {

            htn = htn.trim();

            Taxon higherTaxon = state.getHigherTaxon(htn);

            if (higherTaxon != null){

                higherTaxonNode = higherTaxon.getTaxonNodes().iterator().next();

            }else{

                BotanicalName name = makeHigherTaxonName(state, htn);

                Reference sec = state.getSecReference();

                higherTaxon = Taxon.NewInstance(name, sec);

                higherTaxonNode = rootNode.addChildTaxon(higherTaxon, sec, null);

                state.putHigherTaxon(htn, higherTaxon);

                rootNode = higherTaxonNode;

            }

        }

        return higherTaxonNode;

    }

    private BotanicalName makeHigherTaxonName(IAPTImportState state, String name) {

        // Abteilung: -phyta (bei Pflanzen), -mycota (bei Pilzen)

        // Unterabteilung: -phytina (bei Pflanzen), -mycotina (bei Pilzen)

        // Klasse: -opsida (bei Pflanzen), -phyceae (bei Algen), -mycetes (bei Pilzen)

        // Unterklasse: -idae (bei Pflanzen), -phycidae (bei Algen), -mycetidae (bei Pilzen)

        // Ordnung: -ales

        // Unterordnung: -ineae

        // Familie: -aceae

        // Unterfamilie: -oideae

        // Tribus: -eae

        // Subtribus: -inae

        Rank rank = Rank.UNKNOWN_RANK();

        if(name.matches("phyta$|mycota$")){

            rank = Rank.SECTION_BOTANY();

        } else if(name.matches("phytina$|mycotina$")){

            rank = Rank.SUBSECTION_BOTANY();

        } else if(name.matches("opsida$|phyceae$|mycetes$")){

            rank = Rank.CLASS();

        } else if(name.matches("idae$|phycidae$|mycetidae$")){

            rank = Rank.SUBCLASS();

        } else if(name.matches("ales$")){

            rank = Rank.ORDER();

        } else if(name.matches("ineae$")){

            rank = Rank.SUBORDER();

        } else if(name.matches("aceae$")){

            rank = Rank.FAMILY();

        } else if(name.matches("oideae$")){

            rank = Rank.SUBFAMILY();

        } else if(name.matches("eae$")){

            rank = Rank.TRIBE();

        } else if(name.matches("inae$")){

            rank = Rank.SUBTRIBE();

        }

        BotanicalName taxonName = BotanicalName.NewInstance(rank);

        taxonName.addSource(makeOriginalSource(state));

        taxonName.setGenusOrUninomial(StringUtils.capitalize(name));

        return taxonName;

    }

    /**

     * @param state

     * @return

     */

    private IdentifiableSource makeOriginalSource(IAPTImportState state) {

        return IdentifiableSource.NewDataImportInstance("line: " + state.getCurrentLine(), null, state.getConfig().getSourceReference());

    }

    private Reference makeReference(IAPTImportState state, UUID uuidRef) {

        Reference ref = state.getReference(uuidRef);

        if (ref == null){

            ref = getReferenceService().find(uuidRef);

            state.putReference(uuidRef, ref);

        }

        return ref;

    }

}

// $Id$

/**

* Copyright (C) 2016 EDIT

* European Distributed Institute of Taxonomy

* http://www.e-taxonomy.eu

*

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.io.iapt;

import eu.etaxonomy.cdm.database.ICdmDataSource;

import eu.etaxonomy.cdm.io.common.ImportStateBase;

import eu.etaxonomy.cdm.io.common.mapping.IInputTransformer;

import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;

import eu.etaxonomy.cdm.io.mexico.MexicoBorhidiExcelImport;

import eu.etaxonomy.cdm.io.mexico.MexicoConabioTransformer;

import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;

import eu.etaxonomy.cdm.model.name.NomenclaturalCode;

import eu.etaxonomy.cdm.model.reference.Reference;

import java.net.URI;

/**

 * @author a.mueller

 * @date 16.06.2016

 *

 */

public class IAPTImportConfigurator extends ExcelImportConfiguratorBase{

    private static final long serialVersionUID = -4793138681632122831L;

    private static IInputTransformer defaultTransformer = new IAPTTransformer();

    private Reference secReference;

    public static IAPTImportConfigurator NewInstance(URI source, ICdmDataSource destination) {

        return new IAPTImportConfigurator(source, destination);

    }

    private IAPTImportConfigurator(URI source, ICdmDataSource destination) {

        super(source, destination, defaultTransformer);

        setNomenclaturalCode(NomenclaturalCode.ICNAFP);

        setSource(source);

        setDestination(destination);

     }

    @Override

    public ImportStateBase getNewState() {

        return new SimpleExcelTaxonImportState<>(this);

    }

    @Override

    protected void makeIoClassList() {

        ioClassList = new Class[]{

                IAPTExcelImport.class

        };

    }

    /**

     * @return the secReference

     */

    public Reference getSecReference() {

        return secReference;

    }

    /**

     * @param secReference

     */

    public void setSecReference(Reference secReference) {

        this.secReference = secReference;

    }

}

// $Id$

/**

* Copyright (C) 2007 EDIT

* European Distributed Institute of Taxonomy

* http://www.e-taxonomy.eu

*

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.io.iapt;

import java.util.HashMap;

import java.util.Map;

import java.util.UUID;

import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;

import org.apache.log4j.Logger;

import eu.etaxonomy.cdm.io.excel.common.ExcelImportState;

import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase;

import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;

import eu.etaxonomy.cdm.model.name.BotanicalName;

import eu.etaxonomy.cdm.model.reference.Reference;

import eu.etaxonomy.cdm.model.taxon.Classification;

import eu.etaxonomy.cdm.model.taxon.Taxon;

import eu.etaxonomy.cdm.model.taxon.TaxonNode;

/**

 * @author a.mueller

 * @created 11.05.2009

 */

public class IAPTImportState extends SimpleExcelTaxonImportState<IAPTImportConfigurator> {

	@SuppressWarnings("unused")

	private static final Logger logger = Logger.getLogger(IAPTImportState.class);

	private final Map<String, Taxon> higherTaxonTaxonMap = new HashMap<String, Taxon>();

	private final Map<String, UUID> higherTaxonUuidMap = new HashMap<String, UUID>();

	private final Map<String, BotanicalName> familyNameMap = new HashMap<String, BotanicalName>();

	//classification

	private Classification classification;

    public Classification getClassification() {return classification;}

    public void setClassification(Classification classification) {this.classification = classification;}

    //current taxon

    private Taxon currentTaxon;

    public Taxon getCurrentTaxon() {return currentTaxon;}

    public void setCurrentTaxon(Taxon currentTaxon) {this.currentTaxon = currentTaxon;}

    //rootNode

    private TaxonNode rootNode;

    public void setRootNode(TaxonNode rootNode) {this.rootNode = rootNode;}

    public TaxonNode getRootNode() { return rootNode;}

    private Reference secReference;

    public Reference getSecReference() {return secReference;}

    public void setSecReference(Reference secReference) {this.secReference = secReference;}

    private PresenceAbsenceTerm highestStatusForTaxon;

    public PresenceAbsenceTerm getHighestStatusForTaxon(){return highestStatusForTaxon;}

    public void setHighestStatusForTaxon(PresenceAbsenceTerm highestStatusForTaxon){this.highestStatusForTaxon = highestStatusForTaxon;}

    //Constructor

    public IAPTImportState(IAPTImportConfigurator config) {

		super(config);

	}

    //higher taxon

    public Taxon getHigherTaxon(String higherName) {

        return higherTaxonTaxonMap.get(higherName);

    }

	public Taxon putHigherTaxon(String higherName, Taxon taxon) {

		return higherTaxonTaxonMap.put(higherName, taxon);

	}

	public Taxon removeHigherTaxon(String higherName) {

		return higherTaxonTaxonMap.remove(higherName);

	}

    public boolean containsHigherTaxon(String higherName) {

        return higherTaxonTaxonMap.containsKey(higherName);

    }

    //higher taxon uuid

    public UUID getHigherTaxonUuid(String higherName) {

        return higherTaxonUuidMap.get(higherName);

    }

	public UUID putHigherTaxon(String higherName, UUID uuid) {

		return higherTaxonUuidMap.put(higherName, uuid);

	}

	public UUID removeHigherTaxonUuid(String higherName) {

		return higherTaxonUuidMap.remove(higherName);

	}

    public boolean containsHigherTaxonUuid(String higherName) {

        return higherTaxonUuidMap.containsKey(higherName);

    }

    //family names

    public BotanicalName getFamilyName(String familyStr) {

        return familyNameMap.get(familyStr);

    }

    public void putFamilyName(String familyStr, BotanicalName name) {

        familyNameMap.put(familyStr, name);

    }

    Map<UUID, Reference> refMap = new HashMap<UUID, Reference>();

    //reference

    public Reference getReference(UUID uuidRef) {

        return refMap.get(uuidRef);

    }

    public void putReference(UUID uuidRef, Reference ref) {

        refMap.put(uuidRef, ref);

    }

}

// $Id$

/**

* Copyright (C) 2007 EDIT

* European Distributed Institute of Taxonomy

* http://www.e-taxonomy.eu

*

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.io.iapt;

import java.util.UUID;

import org.apache.log4j.Logger;

import eu.etaxonomy.cdm.io.common.mapping.InputTransformerBase;

/**

 * @author a.mueller

 * @created 01.03.2010

 */

public final class IAPTTransformer extends InputTransformerBase {

    private static final long serialVersionUID = 1070018208741186271L;

    @SuppressWarnings("unused")

	private static final Logger logger = Logger.getLogger(IAPTTransformer.class);

    //references

    public static final UUID uuidRefFRC = UUID.fromString("c1caf6a2-5083-4f44-8f97-9abe23a84cd8");

    public static final UUID uuidRefAS = UUID.fromString("1f15291a-b4c5-4e15-960f-d0145a250539");

    public static final UUID uuidRefFC = UUID.fromString("c5a0bfb8-85b2-422d-babe-423aa2e24c35");

}

import eu.etaxonomy.cdm.model.taxon.Classification;