EDIT

/**

 * Copyright (C) 2007 EDIT

 * European Distributed Institute of Taxonomy

 * http://www.e-taxonomy.eu

 *

 * The contents of this file are subject to the Mozilla Public License Version 1.1

 * See LICENSE.TXT at the top of this package for the full license terms.

 */

package eu.etaxonomy.cdm.io.iapt;

import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;

import eu.etaxonomy.cdm.common.CdmUtils;

import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImport;

import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;

import eu.etaxonomy.cdm.model.common.*;

import eu.etaxonomy.cdm.model.name.*;

import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;

import eu.etaxonomy.cdm.model.occurrence.FieldUnit;

import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;

import eu.etaxonomy.cdm.model.reference.Reference;

import eu.etaxonomy.cdm.model.taxon.*;

import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;

import org.apache.commons.lang.ArrayUtils;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.log4j.Level;

import org.apache.log4j.Logger;

import org.joda.time.DateTimeFieldType;

import org.joda.time.Partial;

import org.springframework.stereotype.Component;

import java.util.*;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

/**

 * @author a.mueller

 * @created 05.01.2016

 */

@Component("iAPTExcelImport")

public class IAPTExcelImport<CONFIG extends IAPTImportConfigurator> extends SimpleExcelTaxonImport<CONFIG> {

    private static final long serialVersionUID = -747486709409732371L;

    private static final Logger logger = Logger.getLogger(IAPTExcelImport.class);

    public static final String ANNOTATION_MARKER_STRING = "[*]";

    private static UUID ROOT_UUID = UUID.fromString("4137fd2a-20f6-4e70-80b9-f296daf51d82");

    private static NonViralNameParserImpl nameParser = NonViralNameParserImpl.NewInstance();

    private final static String REGISTRATIONNO_PK= "RegistrationNo_Pk";

    private final static String HIGHERTAXON= "HigherTaxon";

    private final static String FULLNAME= "FullName";

    private final static String AUTHORSSPELLING= "AuthorsSpelling";

    private final static String LITSTRING= "LitString";

    private final static String REGISTRATION= "Registration";

    private final static String TYPE= "Type";

    private final static String CAVEATS= "Caveats";

    private final static String FULLBASIONYM= "FullBasionym";

    private final static String FULLSYNSUBST= "FullSynSubst";

    private final static String NOTESTXT= "NotesTxt";

    private final static String REGDATE= "RegDate";

    private final static String NAMESTRING= "NameString";

    private final static String BASIONYMSTRING= "BasionymString";

    private final static String SYNSUBSTSTR= "SynSubstStr";

    private final static String AUTHORSTRING= "AuthorString";

    private  static List<String> expectedKeys= Arrays.asList(new String[]{

            REGISTRATIONNO_PK, HIGHERTAXON, FULLNAME, AUTHORSSPELLING, LITSTRING, REGISTRATION, TYPE, CAVEATS, FULLBASIONYM, FULLSYNSUBST, NOTESTXT, REGDATE, NAMESTRING, BASIONYMSTRING, SYNSUBSTSTR, AUTHORSTRING});

    private static final Pattern nomRefTokenizeP = Pattern.compile("^(.*):\\s([^\\.:]+)\\.(.*?)\\.?$");

    private static final Pattern[] nomRefPubDatePs = new Pattern[]{

            // NOTE:

            // The order of the patterns is extremely important!!!

            //

            // all patterns cover the years 1700 - 1999

            Pattern.compile("^(?<year>1[7,8,9][0-9]{2})$"), // only year, like '1969'

            Pattern.compile("^(?<monthName>\\p{L}+\\.?)\\s(?<day>[0-9]{1,2})(?:st|rd|th)?\\.?,?\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like April 12, 1969 or april 12th 1999

            Pattern.compile("^(?<monthName>\\p{L}+\\.?),?\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // April 99 or April, 1999 or Apr. 12

            Pattern.compile("^(?<day>[0-9]{1,2})([\\.\\-/])(?<month>[0-1]?[0-9])\\2(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like 12.04.1969 or 12/04/1969 or 12-04-1969

            Pattern.compile("^(?<month>[0-1]?[0-9])([\\.\\-/])(?<year>(?:1[7,8,9])?[0-9]{2})$"), // partial date like 04.1969 or 04/1969 or 04-1969

            Pattern.compile("^(?<day>[0-9]{1,2})(?:[\\./]|th|rd)?\\s(?<monthName>\\p{L}+\\.?),?\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like 12. April 1969 or april 1999 or 22 Dec.1999

        };

    private static final Pattern typeSplitPattern =  Pattern.compile("^(?:\"*[Tt]ype: (?<type>.*?))(?:[Hh]olotype:(?<holotype>.*?))?(?:[Ii]sotype[^:]*:(?<isotype>.*))?$");

    private static Map<String, Integer> monthFromNameMap = new HashMap<>();

    static {

        String[] ck = new String[]{"leden", "únor", "březen", "duben", "květen", "červen", "červenec ", "srpen", "září", "říjen", "listopad", "prosinec"};

        String[] fr = new String[]{"janvier", "février", "mars", "avril", "mai", "juin", "juillet", "août", "septembre", "octobre", "novembre", "décembre"};

        String[] de = new String[]{"januar", "februar", "märz", "april", "mai", "juni", "juli", "august", "september", "oktober", "november", "dezember"};

        String[] en = new String[]{"january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"};

        String[] it = new String[]{"gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio", "agosto", "settembre", "ottobre", "novembre", "dicembre"};

        String[] sp = new String[]{"enero", "febrero", "marzo", "abril", "mayo", "junio", "julio", "agosto", "septiembre", "octubre", "noviembre", "diciembre"};

        String[] de_abbrev = new String[]{"jan.", "feb.", "märz", "apr.", "mai", "jun.", "jul.", "aug.", "sept.", "okt.", "nov.", "dez."};

        String[] en_abbrev = new String[]{"jan.", "feb.", "mar.", "apr.", "may", "jun.", "jul.", "aug.", "sep.", "oct.", "nov.", "dec."};

        String[] port = new String[]{"Janeiro", "Fevereiro", "Março", "Abril", "Maio", "Junho", "Julho", "Agosto", "Setembro", "Outubro", "Novembro", "Dezembro"};

        String[] rom_num = new String[]{"i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix", "x", "xi", "xii"};

        String[][] perLang =  new String[][]{ck, de, fr, en, it, sp, port, de_abbrev, en_abbrev, rom_num};

        for (String[] months: perLang) {

            for(int m = 1; m < 13; m++){

                monthFromNameMap.put(months[m - 1].toLowerCase(), m);

            }

        }

        // special cases

        monthFromNameMap.put("mar", 3);

        monthFromNameMap.put("dec", 12);

        monthFromNameMap.put("Februari", 2);

    }

    enum TypesName {

        type, holotype, isotype;

        public SpecimenTypeDesignationStatus status(){

            switch (this) {

                case holotype:

                    return SpecimenTypeDesignationStatus.HOLOTYPE();

                case isotype:

                    return SpecimenTypeDesignationStatus.ISOTYPE();

                default:

                    return null;

            }

        }

    }

    private MarkerType markerTypeFossil = null;

    private Rank rankUnrankedSupraGeneric = null;

    private Rank familyIncertisSedis = null;

    private AnnotationType annotationTypeCaveats = null;

    private Taxon makeTaxon(HashMap<String, String> record, SimpleExcelTaxonImportState<CONFIG> state,

                            TaxonNode higherTaxonNode, boolean isFossil) {

        String line = state.getCurrentLine() + ": ";

        String regNumber = getValue(record, REGISTRATIONNO_PK, false);

        String titleCacheStr = getValue(record, FULLNAME, true);

        String nameStr = getValue(record, NAMESTRING, true);

        String authorStr = getValue(record, AUTHORSTRING, true);

        String nomRefStr = getValue(record, LITSTRING, true);

        String authorsSpelling = getValue(record, AUTHORSSPELLING, true);

        String notesTxt = getValue(record, NOTESTXT, true);

        String caveats = getValue(record, CAVEATS, true);

        String fullSynSubstStr = getValue(record, FULLSYNSUBST, true);

        String synSubstStr = getValue(record, SYNSUBSTSTR, true);

        String typeStr = getValue(record, TYPE, true);

        String nomRefTitle = null;

        String nomRefDetail = null;

        String nomRefPupDate = null;

        String nomRefPupDay = null;

        String nomRefPupMonth = null;

        String nomRefPupMonthName = null;

        String nomRefPupYear = null;

        // preprocess nomRef: separate citation, reference detail, publishing date

        if(!StringUtils.isEmpty(nomRefStr)){

            nomRefStr = nomRefStr.trim();

            Matcher m = nomRefTokenizeP.matcher(nomRefStr);

            if(m.matches()){

                nomRefTitle = m.group(1);

                nomRefDetail = m.group(2);

                nomRefPupDate = m.group(3).trim();

                // nomRefDetail.replaceAll("[\\:\\.\\s]", ""); // TODO integrate into nomRefTokenizeP

                for(Pattern p : nomRefPubDatePs){

                    Matcher m2 = p.matcher(nomRefPupDate);

                    if(m2.matches()){

                        try {

                            nomRefPupYear = m2.group("year");

                        } catch (IllegalArgumentException e){

                            // named capture group not found

                        }

                        try {

                            nomRefPupMonth = m2.group("month");

                        } catch (IllegalArgumentException e){

                            // named capture group not found

                        }

                        try {

                            nomRefPupMonthName = m2.group("monthName");

                            nomRefPupMonth = monthFromName(nomRefPupMonthName, regNumber);

                        } catch (IllegalArgumentException e){

                            // named capture group not found

                        }

                        try {

                            nomRefPupDay = m2.group("day");

                        } catch (IllegalArgumentException e){

                            // named capture group not found

                        }

                        if(nomRefPupYear == null){

                            logger.error("nomRefPupYear in " + nomRefStr + " is  NULL" );

                        }

                        if(nomRefPupYear.length() == 2 ){

                            // it is an abbreviated year from the 19** years

                            nomRefPupYear = "19" + nomRefPupYear;

                        }

                        nomRefTitle = nomRefTitle + ": " + nomRefDetail + ". " + nomRefPupYear + ".";

                        break;

                    }

                }

                if(nomRefPupYear == null){

                    logger.warn("Pub date not found in [" + regNumber + "]: " + nomRefPupDate + " from " + nomRefStr );

                }

                List<DateTimeFieldType> types = new ArrayList<>();

                List<Integer> values = new ArrayList<>();

                if(nomRefPupYear != null){

                    types.add(DateTimeFieldType.year());

                    values.add(Integer.parseInt(nomRefPupYear));

                }

                if(nomRefPupMonth != null){

                    types.add(DateTimeFieldType.monthOfYear());

                    values.add(Integer.parseInt(nomRefPupMonth));

                }

                if(nomRefPupDay != null){

                    types.add(DateTimeFieldType.dayOfMonth());

                    values.add(Integer.parseInt(nomRefPupDay));

                }

                Partial pupDate = new Partial(types.toArray(new DateTimeFieldType[types.size()]), ArrayUtils.toPrimitive(values.toArray(new Integer[values.size()])));

            } else {

                nomRefTitle = nomRefStr;

            }

        }

        BotanicalName taxonName = makeBotanicalName(state, titleCacheStr, nameStr, authorStr, nomRefTitle);

        if(!StringUtils.isEmpty(notesTxt)){

            notesTxt = notesTxt.replace("Notes: ", "").trim();

            taxonName.addAnnotation(Annotation.NewInstance(notesTxt, AnnotationType.EDITORIAL(), Language.DEFAULT()));

        }

        if(!StringUtils.isEmpty(caveats)){

            caveats = caveats.replace("Caveats: ", "").trim();

            taxonName.addAnnotation(Annotation.NewInstance(caveats, annotationTypeCaveats(), Language.DEFAULT()));

        }

        //

        // Namerelations

        if(!StringUtils.isEmpty(authorsSpelling)){

            authorsSpelling = authorsSpelling.replaceFirst("Author's spelling:", "").replaceAll("\"", "").trim();

            String[] authorSpellingTokens = StringUtils.split(authorsSpelling, " ");

            String[] nameStrTokens = StringUtils.split(nameStr, " ");

            ArrayUtils.reverse(authorSpellingTokens);

            ArrayUtils.reverse(nameStrTokens);

            for (int i = 0; i < nameStrTokens.length; i++){

                if(i < authorSpellingTokens.length){

                    nameStrTokens[i] = authorSpellingTokens[i];

                }

            }

            ArrayUtils.reverse(nameStrTokens);

            String misspelledNameStr = StringUtils.join (nameStrTokens, ' ');

            // build the fullnameString of the misspelled name

            misspelledNameStr = taxonName.getTitleCache().replace(nameStr, misspelledNameStr);

            TaxonNameBase misspelledName = (BotanicalName) nameParser.parseReferencedName(misspelledNameStr, NomenclaturalCode.ICNAFP, null);

            misspelledName.addRelationshipToName(taxonName, NameRelationshipType.MISSPELLING(), null);

            getNameService().save(misspelledName);

        }

        // Replaced Synonyms

        if(!StringUtils.isEmpty(fullSynSubstStr)){

            fullSynSubstStr = fullSynSubstStr.replace("Syn. subst.: ", "");

            BotanicalName replacedSynonymName = makeBotanicalName(state, fullSynSubstStr, synSubstStr, null, null);

            replacedSynonymName.addReplacedSynonym(taxonName, null, null, null);

            getNameService().save(replacedSynonymName);

        }

        Reference sec = state.getConfig().getSecReference();

        Taxon taxon = Taxon.NewInstance(taxonName, sec);

        // Markers

        if(isFossil){

            taxon.addMarker(Marker.NewInstance(markerTypeFossil(), true));

        }

        // Types

        if(!StringUtils.isEmpty(typeStr)){

            Matcher m = typeSplitPattern.matcher(typeStr);

            if(m.matches()){

                String typeString = m.group(TypesName.type.name());

                boolean isFieldUnit = typeStr.matches(".*([°']|\\d+\\s?m\\s|\\d+\\s?km\\s).*"); // check for location or unit m, km

                if(isFieldUnit) {

                    // type as fieldUnit

                    FieldUnit fu = FieldUnit.NewInstance();

                    fu.setTitleCache(typeString, true);

                    getOccurrenceService().save(fu);

                    // all others ..

                    addSpecimenTypes(taxonName, fu, m.group(TypesName.holotype.name()), TypesName.holotype, false);

                    addSpecimenTypes(taxonName, fu, m.group(TypesName.isotype.name()), TypesName.isotype, true);

                } else {

                    TaxonNameBase typeName = nameParser.parseFullName(typeString);

                    taxonName.addNameTypeDesignation(typeName, null, null, null, NameTypeDesignationStatus.AUTOMATIC(), true, true, true, true);

                }

            }

            getNameService().save(taxonName);

        }

        getTaxonService().save(taxon);

        if(higherTaxonNode != null){

            higherTaxonNode.addChildTaxon(taxon, null, null);

            getTaxonNodeService().save(higherTaxonNode);

        }

        return taxon;

    }

    private String monthFromName(String monthName, String regNumber) {

        Integer month = monthFromNameMap.get(monthName.toLowerCase());

        if(month == null){

            logger.warn("Unknown month [" + regNumber + "]: " + monthName + " (" + monthName.toLowerCase() + ")");

            return null;

        } else {

            return month.toString();

        }

    }

    private void addSpecimenTypes(BotanicalName taxonName, FieldUnit fieldUnit, String typeStr, TypesName typeName, boolean multiple){

        if(StringUtils.isEmpty(typeStr)){

            return;

        }

        typeStr = typeStr.trim().replaceAll("\\.$", "");

        List<String> typeData = new ArrayList<>();

        if(multiple){

            String[] tokens = typeStr.split("\\s?,\\s?");

            for (String t : tokens) {

                if(!t.isEmpty()){

                    typeData.add(t.trim());

                }

            }

        } else {

            typeData.add(typeStr.trim());

        }

        for(String type : typeData){

            DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(SpecimenOrObservationType.OtherSpecimen, fieldUnit);

            facade.setTitleCache(type, true);

            DerivedUnit specimen = facade.innerDerivedUnit();

            taxonName.addSpecimenTypeDesignation(specimen, typeName.status(), null, null, null, false, true);

       }

    }

    private BotanicalName makeBotanicalName(SimpleExcelTaxonImportState<CONFIG> state, String titleCacheStr, String nameStr,

                                            String authorStr, String nomRefTitle) {

        BotanicalName taxonName;// cache field for the taxonName.titleCache

        String taxonNameTitleCache = null;

        Map<String, AnnotationType> nameAnnotations = new HashMap<>();

        String line = state.getCurrentLine() + ": ";

        // TitleCache preprocessing

        if(titleCacheStr.endsWith(ANNOTATION_MARKER_STRING) || (authorStr != null && authorStr.endsWith(ANNOTATION_MARKER_STRING))){

            nameAnnotations.put("Author abbreviation not checked.", AnnotationType.EDITORIAL());

            titleCacheStr = titleCacheStr.replace(ANNOTATION_MARKER_STRING, "").trim();

            authorStr = authorStr.replace(ANNOTATION_MARKER_STRING, "").trim();

        }

        // parse the full taxon name

        if(!StringUtils.isEmpty(nomRefTitle)){

            String referenceSeparator = nomRefTitle.startsWith("in ") ? " " : ", ";

            String taxonFullNameStr = titleCacheStr + referenceSeparator + nomRefTitle;

            logger.debug(":::::" + taxonFullNameStr);

            taxonName = (BotanicalName) nameParser.parseReferencedName(taxonFullNameStr, NomenclaturalCode.ICNAFP, null);

        } else {

            taxonName = (BotanicalName) nameParser.parseFullName(titleCacheStr, NomenclaturalCode.ICNAFP, null);

        }

        taxonNameTitleCache = taxonName.getTitleCache().trim();

        if (taxonName.isProtectedTitleCache()) {

            logger.warn(line + "Name could not be parsed: " + titleCacheStr);

        } else {

            boolean doRestoreTitleCacheStr = false;

            // Check if titleCache and nameCache are plausible

            String titleCacheCompareStr = titleCacheStr;

            String nameCache = taxonName.getNameCache();

            String nameCompareStr = nameStr;

            if(taxonName.isBinomHybrid()){

                titleCacheCompareStr = titleCacheCompareStr.replace(" x ", " ×");

                nameCompareStr = nameCompareStr.replace(" x ", " ×");

            }

            if(taxonName.isMonomHybrid()){

                titleCacheCompareStr = titleCacheCompareStr.replaceAll("^X ", "× ");

                nameCompareStr = nameCompareStr.replace("^X ", "× ");

            }

            if(authorStr != null && authorStr.contains(" et ")){

                titleCacheCompareStr = titleCacheCompareStr.replaceAll(" et ", " & ");

            }

            if (!taxonNameTitleCache.equals(titleCacheCompareStr)) {

                logger.warn(line + "The generated titleCache differs from the imported string : " + taxonNameTitleCache + " <> " + titleCacheStr + " will restore original titleCacheStr");

                doRestoreTitleCacheStr = true;

            }

            if (!nameCache.trim().equals(nameCompareStr)) {

                logger.warn(line + "The parsed nameCache differs from " + NAMESTRING + " : " + nameCache + " <> " + nameCompareStr);

            }

            //  Author

            //nameParser.handleAuthors(taxonName, titleCacheStr, authorStr);

            //if (!titleCacheStr.equals(taxonName.getTitleCache())) {

            //    logger.warn(line + "titleCache has changed after setting authors, will restore original titleCacheStr");

            //    doRestoreTitleCacheStr = true;

            //}

            if(doRestoreTitleCacheStr){

                taxonName.setTitleCache(titleCacheStr, true);

            }

            // deduplicate

            replaceAuthorNamesAndNomRef(state, taxonName);

        }

        // Annotations

        if(!nameAnnotations.isEmpty()){

            for(String text : nameAnnotations.keySet()){

                taxonName.addAnnotation(Annotation.NewInstance(text, nameAnnotations.get(text), Language.DEFAULT()));

            }

            getNameService().save(taxonName);

        }

        return taxonName;

    }

    /**

     * @param state

     * @return

     */

    private TaxonNode getClassificationRootNode(IAPTImportState state) {

     //   Classification classification = state.getClassification();

     //   if (classification == null){

     //       IAPTImportConfigurator config = state.getConfig();

     //       classification = Classification.NewInstance(state.getConfig().getClassificationName());

     //       classification.setUuid(config.getClassificationUuid());

     //       classification.setReference(config.getSecReference());

     //       classification = getClassificationService().find(state.getConfig().getClassificationUuid());

     //   }

        TaxonNode rootNode = state.getRootNode();

        if (rootNode == null){

            rootNode = getTaxonNodeService().find(ROOT_UUID);

        }

        if (rootNode == null){

            Classification classification = state.getClassification();

            if (classification == null){

                Reference sec = state.getSecReference();

                String classificationName = state.getConfig().getClassificationName();

                Language language = Language.DEFAULT();

                classification = Classification.NewInstance(classificationName, sec, language);

                state.setClassification(classification);

                classification.setUuid(state.getConfig().getClassificationUuid());

                classification.getRootNode().setUuid(ROOT_UUID);

                getClassificationService().save(classification);

            }

            rootNode = classification.getRootNode();

            state.setRootNode(rootNode);

        }

        return rootNode;

    }

    /**

     * @param record

     * @param originalKey

     * @param doUnescapeHtmlEntities

     * @return

     */

    private String getValue(HashMap<String, String> record, String originalKey, boolean doUnescapeHtmlEntities) {

        String value = record.get(originalKey);

        value = fixCharacters(value);

        if (! StringUtils.isBlank(value)) {

        	if (logger.isDebugEnabled()) {

        	    logger.debug(originalKey + ": " + value);

        	}

        	value = CdmUtils.removeDuplicateWhitespace(value.trim()).toString();

            if(doUnescapeHtmlEntities){

                value = StringEscapeUtils.unescapeHtml(value);

            }

        	return value.trim();

        }else{

        	return null;

        }

    }

    /**

     * Fixes broken characters.

     * For details see

     * http://dev.e-taxonomy.eu/redmine/issues/6035

     *

     * @param value

     * @return

     */

    private String fixCharacters(String value) {

        value = StringUtils.replace(value, "s$K", "š");

        value = StringUtils.replace(value, "n$K", "ň");

        value = StringUtils.replace(value, "e$K", "ě");

        value = StringUtils.replace(value, "r$K", "ř");

        value = StringUtils.replace(value, "c$K", "č");

        value = StringUtils.replace(value, "z$K", "ž");

        value = StringUtils.replace(value, "S>U$K", "Š");

        value = StringUtils.replace(value, "C>U$K", "Č");

        value = StringUtils.replace(value, "R>U$K", "Ř");

        value = StringUtils.replace(value, "Z>U$K", "Ž");

        value = StringUtils.replace(value, "g$K", "ǧ");

        value = StringUtils.replace(value, "s$A", "ś");

        value = StringUtils.replace(value, "n$A", "ń");

        value = StringUtils.replace(value, "c$A", "ć");

        value = StringUtils.replace(value, "e$E", "ę");

        value = StringUtils.replace(value, "o$H", "õ");

        value = StringUtils.replace(value, "s$C", "ş");

        value = StringUtils.replace(value, "t$C", "ț");

        value = StringUtils.replace(value, "S>U$C", "Ş");

        value = StringUtils.replace(value, "a$O", "å");

        value = StringUtils.replace(value, "A>U$O", "Å");

        value = StringUtils.replace(value, "u$O", "ů");

        value = StringUtils.replace(value, "g$B", "ğ");

        value = StringUtils.replace(value, "g$B", "ĕ");

        value = StringUtils.replace(value, "a$B", "ă");

        value = StringUtils.replace(value, "l$/", "ł");

        value = StringUtils.replace(value, ">i", "ı");

        value = StringUtils.replace(value, "i$U", "ï");

        // Special-cases

        value = StringUtils.replace(value, "&yacute", "ý");

        value = StringUtils.replace(value, ">L", "Ł"); // corrected rule

        value = StringUtils.replace(value, "E>U$D", "З");

        value = StringUtils.replace(value, "S>U$E", "Ş");

        value = StringUtils.replace(value, "s$E", "ş");

        value = StringUtils.replace(value, "c$k", "č");

        value = StringUtils.replace(value, " U$K", " Š");

        return value;

    }

    /**

	 *  Stores taxa records in DB

	 */

	@Override

    protected void firstPass(SimpleExcelTaxonImportState<CONFIG> state) {

        String lineNumber = state.getCurrentLine() + ": ";

        logger.setLevel(Level.DEBUG);

        HashMap<String, String> record = state.getOriginalRecord();

        logger.debug(lineNumber + record.toString());

        Set<String> keys = record.keySet();

        for (String key: keys) {

            if (! expectedKeys.contains(key)){

                logger.warn(lineNumber + "Unexpected Key: " + key);

            }

        }

        String reg_id = record.get(REGISTRATIONNO_PK);

        //higherTaxon

        String higherTaxaString = record.get(HIGHERTAXON);

        boolean isFossil = false;

        if(higherTaxaString.startsWith("FOSSIL ")){

            higherTaxaString = higherTaxaString.replace("FOSSIL ", "");

            isFossil = true;

        }

        TaxonNode higherTaxon = getHigherTaxon(higherTaxaString, (IAPTImportState)state);

       //Taxon

        Taxon taxon = makeTaxon(record, state, higherTaxon, isFossil);

        if (taxon == null){

            logger.warn(lineNumber + "taxon could not be created and is null");

            return;

        }

        ((IAPTImportState)state).setCurrentTaxon(taxon);

		return;

    }

    private TaxonNode getHigherTaxon(String higherTaxaString, IAPTImportState state) {

        String[] higherTaxaNames = higherTaxaString.toLowerCase().replaceAll("[\\[\\]]", "").split(":");

        TaxonNode higherTaxonNode = null;

        ITaxonTreeNode rootNode = getClassificationRootNode(state);

        for (String htn :  higherTaxaNames) {

            htn = StringUtils.capitalize(htn.trim());

            Taxon higherTaxon = state.getHigherTaxon(htn);

            if (higherTaxon != null){

                higherTaxonNode = higherTaxon.getTaxonNodes().iterator().next();

            }else{

                BotanicalName name = makeHigherTaxonName(state, htn);

                Reference sec = state.getSecReference();

                higherTaxon = Taxon.NewInstance(name, sec);

                getTaxonService().save(higherTaxon);

                higherTaxonNode = rootNode.addChildTaxon(higherTaxon, sec, null);

                state.putHigherTaxon(htn, higherTaxon);

                getClassificationService().saveTreeNode(higherTaxonNode);

            }

            rootNode = higherTaxonNode;

        }

        return higherTaxonNode;

    }

    private BotanicalName makeHigherTaxonName(IAPTImportState state, String name) {

        Rank rank = guessRank(name);

        BotanicalName taxonName = BotanicalName.NewInstance(rank);

        taxonName.addSource(makeOriginalSource(state));

        taxonName.setGenusOrUninomial(StringUtils.capitalize(name));

        return taxonName;

    }

    private Rank guessRank(String name) {

        // normalize

        name = name.replaceAll("\\(.*\\)", "").trim();

        if(name.matches("^Plantae$|^Fungi$")){

           return Rank.KINGDOM();

        } else if(name.matches("^Incertae sedis$|^No group assigned$")){

           return rankFamilyIncertisSedis();

        } else if(name.matches(".*phyta$|.*mycota$")){

           return Rank.SECTION_BOTANY();

        } else if(name.matches(".*phytina$|.*mycotina$")){

           return Rank.SUBSECTION_BOTANY();

        } else if(name.matches("Gymnospermae$|.*ones$")){ // Monocotyledones, Dicotyledones

            return rankUnrankedSupraGeneric();

        } else if(name.matches(".*opsida$|.*phyceae$|.*mycetes$|.*ones$|^Musci$|^Hepaticae$")){

           return Rank.CLASS();

        } else if(name.matches(".*idae$|.*phycidae$|.*mycetidae$")){

           return Rank.SUBCLASS();

        } else if(name.matches(".*ales$")){

           return Rank.ORDER();

        } else if(name.matches(".*ineae$")){

           return Rank.SUBORDER();

        } else if(name.matches(".*aceae$")){

            return Rank.FAMILY();

        } else if(name.matches(".*oideae$")){

           return Rank.SUBFAMILY();

        } else

        //    if(name.matches(".*eae$")){

        //    return Rank.TRIBE();

        // } else

            if(name.matches(".*inae$")){

           return Rank.SUBTRIBE();

        } else if(name.matches(".*ae$")){

           return Rank.FAMILY();

        }

        return Rank.UNKNOWN_RANK();

    }

    private Rank rankUnrankedSupraGeneric() {

        if(rankUnrankedSupraGeneric == null){

            rankUnrankedSupraGeneric = Rank.NewInstance(RankClass.Suprageneric, "Unranked supra generic", " ", " ");

            getTermService().save(rankUnrankedSupraGeneric);

        }

        return rankUnrankedSupraGeneric;

    }

    private Rank rankFamilyIncertisSedis() {

        if(familyIncertisSedis == null){

            familyIncertisSedis = Rank.NewInstance(RankClass.Suprageneric, "Family incertis sedis", " ", " ");

            getTermService().save(familyIncertisSedis);

        }

        return familyIncertisSedis;

    }

    private AnnotationType annotationTypeCaveats(){

        if(annotationTypeCaveats == null){

            annotationTypeCaveats = AnnotationType.NewInstance("Caveats", "Caveats", "");

            getTermService().save(annotationTypeCaveats);

        }

        return annotationTypeCaveats;

    }

    /**

     * @param state

     * @return

     */

    private IdentifiableSource makeOriginalSource(IAPTImportState state) {

        return IdentifiableSource.NewDataImportInstance("line: " + state.getCurrentLine(), null, state.getConfig().getSourceReference());

    }

    private Reference makeReference(IAPTImportState state, UUID uuidRef) {

        Reference ref = state.getReference(uuidRef);

        if (ref == null){

            ref = getReferenceService().find(uuidRef);

            state.putReference(uuidRef, ref);

        }

        return ref;

    }

    private MarkerType markerTypeFossil(){

        if(this.markerTypeFossil == null){

            markerTypeFossil = MarkerType.NewInstance("isFossilTaxon", "isFossil", null);

            getTermService().save(this.markerTypeFossil);

        }

        return markerTypeFossil;

    }

}