EDIT

/**

 * Copyright (C) 2007 EDIT

 * European Distributed Institute of Taxonomy

 * http://www.e-taxonomy.eu

 *

 * The contents of this file are subject to the Mozilla Public License Version 1.1

 * See LICENSE.TXT at the top of this package for the full license terms.

 */

package eu.etaxonomy.cdm.io.iapt;

import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;

import eu.etaxonomy.cdm.common.CdmUtils;

import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImport;

import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;

import eu.etaxonomy.cdm.model.agent.Institution;

import eu.etaxonomy.cdm.model.common.*;

import eu.etaxonomy.cdm.model.name.*;

import eu.etaxonomy.cdm.model.occurrence.*;

import eu.etaxonomy.cdm.model.occurrence.Collection;

import eu.etaxonomy.cdm.model.reference.Reference;

import eu.etaxonomy.cdm.model.taxon.*;

import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;

import org.apache.commons.lang.ArrayUtils;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.log4j.Level;

import org.apache.log4j.Logger;

import org.joda.time.DateTimeFieldType;

import org.joda.time.Partial;

import org.joda.time.format.DateTimeFormat;

import org.joda.time.format.DateTimeFormatter;

import org.springframework.stereotype.Component;

import java.util.*;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

/**

 * @author a.mueller

 * @created 05.01.2016

 */

@Component("iAPTExcelImport")

public class IAPTExcelImport<CONFIG extends IAPTImportConfigurator> extends SimpleExcelTaxonImport<CONFIG> {

    private static final long serialVersionUID = -747486709409732371L;

    private static final Logger logger = Logger.getLogger(IAPTExcelImport.class);

    public static final String ANNOTATION_MARKER_STRING = "[*]";

    private static UUID ROOT_UUID = UUID.fromString("4137fd2a-20f6-4e70-80b9-f296daf51d82");

    private static NonViralNameParserImpl nameParser = NonViralNameParserImpl.NewInstance();

    private final static String REGISTRATIONNO_PK= "RegistrationNo_Pk";

    private final static String HIGHERTAXON= "HigherTaxon";

    private final static String FULLNAME= "FullName";

    private final static String AUTHORSSPELLING= "AuthorsSpelling";

    private final static String LITSTRING= "LitString";

    private final static String REGISTRATION= "Registration";

    private final static String TYPE= "Type";

    private final static String CAVEATS= "Caveats";

    private final static String FULLBASIONYM= "FullBasionym";

    private final static String FULLSYNSUBST= "FullSynSubst";

    private final static String NOTESTXT= "NotesTxt";

    private final static String REGDATE= "RegDate";

    private final static String NAMESTRING= "NameString";

    private final static String BASIONYMSTRING= "BasionymString";

    private final static String SYNSUBSTSTR= "SynSubstStr";

    private final static String AUTHORSTRING= "AuthorString";

    private  static List<String> expectedKeys= Arrays.asList(new String[]{

            REGISTRATIONNO_PK, HIGHERTAXON, FULLNAME, AUTHORSSPELLING, LITSTRING, REGISTRATION, TYPE, CAVEATS, FULLBASIONYM, FULLSYNSUBST, NOTESTXT, REGDATE, NAMESTRING, BASIONYMSTRING, SYNSUBSTSTR, AUTHORSTRING});

    private static final Pattern nomRefTokenizeP = Pattern.compile("^(.*):\\s([^\\.:]+)\\.(.*?)\\.?$");

    private static final Pattern[] nomRefPubDatePs = new Pattern[]{

            // NOTE:

            // The order of the patterns is extremely important!!!

            //

            // all patterns cover the years 1700 - 1999

            Pattern.compile("^(?<year>1[7,8,9][0-9]{2})$"), // only year, like '1969'

            Pattern.compile("^(?<monthName>\\p{L}+\\.?)\\s(?<day>[0-9]{1,2})(?:st|rd|th)?\\.?,?\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like April 12, 1969 or april 12th 1999

            Pattern.compile("^(?<monthName>\\p{L}+\\.?),?\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // April 99 or April, 1999 or Apr. 12

            Pattern.compile("^(?<day>[0-9]{1,2})([\\.\\-/])(?<month>[0-1]?[0-9])\\2(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like 12.04.1969 or 12/04/1969 or 12-04-1969

            Pattern.compile("^(?<day>[0-9]{1,2})([\\.\\-/])(?<month>[IVX]{1,2})\\2(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like 12-VI-1969

            Pattern.compile("^(?:(?<day>[0-9]{1,2})\\sde\\s)(?<monthName>\\p{L}+)\\sde\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full and partial date like 12 de Enero de 1999 or Enero de 1999

            Pattern.compile("^(?<month>[0-1]?[0-9])([\\.\\-/])(?<year>(?:1[7,8,9])?[0-9]{2})$"), // partial date like 04.1969 or 04/1969 or 04-1969

            Pattern.compile("^(?<year>(?:1[7,8,9])?[0-9]{2})([\\.\\-/])(?<month>[0-1]?[0-9])$"),//  partial date like 1999-04

            Pattern.compile("^(?<day>[0-9]{1,2})(?:[\\./]|th|rd)?\\s(?<monthName>\\p{L}+\\.?),?\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like 12. April 1969 or april 1999 or 22 Dec.1999

        };

    private static final Pattern typeSplitPattern =  Pattern.compile("^(?:\"*[Tt]ype: (?<type>.*?))(?:[Hh]olotype:(?<holotype>.*?)\\.?)?(?:[Ii]sotype[^:]*:(?<isotype>.*)\\.?)?\\.?$");

    // AccessionNumbers: , #.*, n°:?, 96/3293, No..*, -?\w{1,3}-[0-9\-/]*

    private static final Pattern accessionNumberOnlyPattern = Pattern.compile("^(?<accNumber>(?:n°\\:?\\s?|#|No\\.?\\s?)?[\\d\\w\\-/]*)$");

    private static final Pattern[] specimenTypePatterns = new Pattern[]{

            Pattern.compile("^(?<colCode>[A-Z]+|CPC Micropaleontology Lab\\.?)\\s+(?:\\((?<institute>.*[^\\)])\\))(?<accNumber>.*)?$"), // like: GAUF (Gansu Agricultural University) No. 1207-1222

            Pattern.compile("^(?<colCode>[A-Z]+|CPC Micropaleontology Lab\\.?)\\s+(?:Coll\\.\\s(?<subCollection>[^\\.,;]*)(.))(?<accNumber>.*)?$"), // like KASSEL Coll. Krasske, Praep. DII 78

            Pattern.compile("^(?:Coll\\.\\s(?<subCollection>[^\\.,;]*)(.))(?<institute>.*)\\2(?<accNumber>.*)?$"), // like Coll. Lange-Bertalot, Bot. Inst., Univ. Frankfurt/Main, Germany Praep. Neukaledonien OTL 62

            Pattern.compile("^(?<colCode>[A-Z]+)(?:\\s+(?<accNumber>.*))?$"), // identifies the Collection code and takes the rest as accessionNumber if any

    };

    private static Map<String, Integer> monthFromNameMap = new HashMap<>();

    static {

        String[] ck = new String[]{"leden", "únor", "březen", "duben", "květen", "červen", "červenec ", "srpen", "září", "říjen", "listopad", "prosinec"};

        String[] fr = new String[]{"janvier", "février", "mars", "avril", "mai", "juin", "juillet", "août", "septembre", "octobre", "novembre", "décembre"};

        String[] de = new String[]{"januar", "februar", "märz", "april", "mai", "juni", "juli", "august", "september", "oktober", "november", "dezember"};

        String[] en = new String[]{"january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"};

        String[] it = new String[]{"gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio", "agosto", "settembre", "ottobre", "novembre", "dicembre"};

        String[] sp = new String[]{"enero", "febrero", "marzo", "abril", "mayo", "junio", "julio", "agosto", "septiembre", "octubre", "noviembre", "diciembre"};

        String[] de_abbrev = new String[]{"jan.", "feb.", "märz", "apr.", "mai", "jun.", "jul.", "aug.", "sept.", "okt.", "nov.", "dez."};

        String[] en_abbrev = new String[]{"jan.", "feb.", "mar.", "apr.", "may", "jun.", "jul.", "aug.", "sep.", "oct.", "nov.", "dec."};

        String[] port = new String[]{"Janeiro", "Fevereiro", "Março", "Abril", "Maio", "Junho", "Julho", "Agosto", "Setembro", "Outubro", "Novembro", "Dezembro"};

        String[] rom_num = new String[]{"i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix", "x", "xi", "xii"};

        String[][] perLang =  new String[][]{ck, de, fr, en, it, sp, port, de_abbrev, en_abbrev, rom_num};

        for (String[] months: perLang) {

            for(int m = 1; m < 13; m++){

                monthFromNameMap.put(months[m - 1].toLowerCase(), m);

            }

        }

        // special cases

        monthFromNameMap.put("mar", 3);

        monthFromNameMap.put("dec", 12);

        monthFromNameMap.put("Februari", 2);

    }

    DateTimeFormatter formatterYear = DateTimeFormat.forPattern("yyyy");

    private Map<String, Collection> collectionMap = new HashMap<>();

    enum TypesName {

        type, holotype, isotype;

        public SpecimenTypeDesignationStatus status(){

            switch (this) {

                case holotype:

                    return SpecimenTypeDesignationStatus.HOLOTYPE();

                case isotype:

                    return SpecimenTypeDesignationStatus.ISOTYPE();

                default:

                    return null;

            }

        }

    }

    private MarkerType markerTypeFossil = null;

    private Rank rankUnrankedSupraGeneric = null;

    private Rank familyIncertisSedis = null;

    private AnnotationType annotationTypeCaveats = null;

    private Taxon makeTaxon(HashMap<String, String> record, SimpleExcelTaxonImportState<CONFIG> state,

                            TaxonNode higherTaxonNode, boolean isFossil) {

        String line = state.getCurrentLine() + ": ";

        String regNumber = getValue(record, REGISTRATIONNO_PK, false);

        String regStr = getValue(record, REGISTRATION, true);

        String titleCacheStr = getValue(record, FULLNAME, true);

        String nameStr = getValue(record, NAMESTRING, true);

        String authorStr = getValue(record, AUTHORSTRING, true);

        String nomRefStr = getValue(record, LITSTRING, true);

        String authorsSpelling = getValue(record, AUTHORSSPELLING, true);

        String notesTxt = getValue(record, NOTESTXT, true);

        String caveats = getValue(record, CAVEATS, true);

        String fullSynSubstStr = getValue(record, FULLSYNSUBST, true);

        String synSubstStr = getValue(record, SYNSUBSTSTR, true);

        String typeStr = getValue(record, TYPE, true);

        String nomRefTitle = null;

        String nomRefDetail;

        String nomRefPupDate = null;

        Partial pupDate = null;

        // preprocess nomRef: separate citation, reference detail, publishing date

        if(!StringUtils.isEmpty(nomRefStr)){

            nomRefStr = nomRefStr.trim();

            Matcher m = nomRefTokenizeP.matcher(nomRefStr);

            if(m.matches()){

                nomRefTitle = m.group(1);

                nomRefDetail = m.group(2);

                nomRefPupDate = m.group(3).trim();

                pupDate = parsePubDate(regNumber, nomRefStr, nomRefPupDate);

                if (pupDate != null) {

                    nomRefTitle = nomRefTitle + ": " + nomRefDetail + ". " + pupDate.toString(formatterYear) + ".";

                }

            } else {

                nomRefTitle = nomRefStr;

            }

        }

        BotanicalName taxonName = makeBotanicalName(state, regNumber, titleCacheStr, nameStr, authorStr, nomRefTitle);

        // always add the original strings of parsed data as annotation

        taxonName.addAnnotation(Annotation.NewInstance("imported and parsed data strings:" +

                        "\n -  '" + LITSTRING + "': "+ nomRefStr +

                        "\n -  '" + TYPE + "': " + typeStr +

                        "\n -  '" + REGISTRATION  + "': " + regStr

                , AnnotationType.TECHNICAL(), Language.DEFAULT()));

        if(pupDate != null) {

            taxonName.getNomenclaturalReference().setDatePublished(TimePeriod.NewInstance(pupDate));

        }

        if(!StringUtils.isEmpty(notesTxt)){

            notesTxt = notesTxt.replace("Notes: ", "").trim();

            taxonName.addAnnotation(Annotation.NewInstance(notesTxt, AnnotationType.EDITORIAL(), Language.DEFAULT()));

        }

        if(!StringUtils.isEmpty(caveats)){

            caveats = caveats.replace("Caveats: ", "").trim();

            taxonName.addAnnotation(Annotation.NewInstance(caveats, annotationTypeCaveats(), Language.DEFAULT()));

        }

        //

        // Namerelations

        if(!StringUtils.isEmpty(authorsSpelling)){

            authorsSpelling = authorsSpelling.replaceFirst("Author's spelling:", "").replaceAll("\"", "").trim();

            String[] authorSpellingTokens = StringUtils.split(authorsSpelling, " ");

            String[] nameStrTokens = StringUtils.split(nameStr, " ");

            ArrayUtils.reverse(authorSpellingTokens);

            ArrayUtils.reverse(nameStrTokens);

            for (int i = 0; i < nameStrTokens.length; i++){

                if(i < authorSpellingTokens.length){

                    nameStrTokens[i] = authorSpellingTokens[i];

                }

            }

            ArrayUtils.reverse(nameStrTokens);

            String misspelledNameStr = StringUtils.join (nameStrTokens, ' ');

            // build the fullnameString of the misspelled name

            misspelledNameStr = taxonName.getTitleCache().replace(nameStr, misspelledNameStr);

            TaxonNameBase misspelledName = (BotanicalName) nameParser.parseReferencedName(misspelledNameStr, NomenclaturalCode.ICNAFP, null);

            misspelledName.addRelationshipToName(taxonName, NameRelationshipType.MISSPELLING(), null);

            getNameService().save(misspelledName);

        }

        // Replaced Synonyms

        if(!StringUtils.isEmpty(fullSynSubstStr)){

            fullSynSubstStr = fullSynSubstStr.replace("Syn. subst.: ", "");

            BotanicalName replacedSynonymName = makeBotanicalName(state, regNumber, fullSynSubstStr, synSubstStr, null, null);

            replacedSynonymName.addReplacedSynonym(taxonName, null, null, null);

            getNameService().save(replacedSynonymName);

        }

        Reference sec = state.getConfig().getSecReference();

        Taxon taxon = Taxon.NewInstance(taxonName, sec);

        // Markers

        if(isFossil){

            taxon.addMarker(Marker.NewInstance(markerTypeFossil(), true));

        }

        // Types

        if(!StringUtils.isEmpty(typeStr)){

            makeTypeData(typeStr, taxonName, regNumber);

        }

        getTaxonService().save(taxon);

        if(higherTaxonNode != null){

            higherTaxonNode.addChildTaxon(taxon, null, null);

            getTaxonNodeService().save(higherTaxonNode);

        }

        return taxon;

    }

    private void makeTypeData(String typeStr, BotanicalName taxonName, String regNumber) {

        Matcher m = typeSplitPattern.matcher(typeStr);

        if(m.matches()){

            String typeString = m.group(TypesName.type.name());

            boolean isFieldUnit = typeStr.matches(".*([°']|\\d+\\s?m\\s|\\d+\\s?km\\s).*"); // check for location or unit m, km

            if(isFieldUnit) {

                // type as fieldUnit

                FieldUnit fu = FieldUnit.NewInstance();

                fu.setTitleCache(typeString, true);

                getOccurrenceService().save(fu);

                // all others ..

                addSpecimenTypes(taxonName, fu, m.group(TypesName.holotype.name()), TypesName.holotype, false, regNumber);

                addSpecimenTypes(taxonName, fu, m.group(TypesName.isotype.name()), TypesName.isotype, true, regNumber);

            } else {

                TaxonNameBase typeName = nameParser.parseFullName(typeString);

                taxonName.addNameTypeDesignation(typeName, null, null, null, NameTypeDesignationStatus.AUTOMATIC(), true, true, true, true);

            }

        }

        getNameService().save(taxonName);

    }

    private Partial parsePubDate(String regNumber, String nomRefStr, String nomRefPupDate) {

        Partial pupDate = null;

        boolean parseError = false;

        String nomRefPupDay = null;

        String nomRefPupMonth = null;

        String nomRefPupMonthName = null;

        String nomRefPupYear = null;

        // nomRefDetail.replaceAll("[\\:\\.\\s]", ""); // TODO integrate into nomRefTokenizeP

        for(Pattern p : nomRefPubDatePs){

            Matcher m2 = p.matcher(nomRefPupDate);

            if(m2.matches()){

                try {

                    nomRefPupYear = m2.group("year");

                } catch (IllegalArgumentException e){

                    // named capture group not found

                }

                try {

                    nomRefPupMonth = m2.group("month");

                } catch (IllegalArgumentException e){

                    // named capture group not found

                }

                try {

                    nomRefPupMonthName = m2.group("monthName");

                    nomRefPupMonth = monthFromName(nomRefPupMonthName, regNumber);

                    if(nomRefPupMonth == null){

                        parseError = true;

                    }

                } catch (IllegalArgumentException e){

                    // named capture group not found

                }

                try {

                    nomRefPupDay = m2.group("day");

                } catch (IllegalArgumentException e){

                    // named capture group not found

                }

                if(nomRefPupYear == null){

                    logger.error("nomRefPupYear in " + nomRefStr + " is  NULL" );

                    parseError = true;

                }

                if(nomRefPupYear.length() == 2 ){

                    // it is an abbreviated year from the 19** years

                    nomRefPupYear = "19" + nomRefPupYear;

                }

                break;

            }

        }

        if(nomRefPupYear == null){

            logger.warn(csvReportLine(regNumber, "Pub date", nomRefPupDate, "in", nomRefStr, "not parsable"));

            parseError = true;

        }

        List<DateTimeFieldType> types = new ArrayList<>();

        List<Integer> values = new ArrayList<>();

        if(!parseError) {

            types.add(DateTimeFieldType.year());

            values.add(Integer.parseInt(nomRefPupYear));

            if (nomRefPupMonth != null) {

                types.add(DateTimeFieldType.monthOfYear());

                values.add(Integer.parseInt(nomRefPupMonth));

            }

            if (nomRefPupDay != null) {

                types.add(DateTimeFieldType.dayOfMonth());

                values.add(Integer.parseInt(nomRefPupDay));

            }

            pupDate = new Partial(types.toArray(new DateTimeFieldType[types.size()]), ArrayUtils.toPrimitive(values.toArray(new Integer[values.size()])));

        }

        return pupDate;

    }

    private String monthFromName(String monthName, String regNumber) {

        Integer month = monthFromNameMap.get(monthName.toLowerCase());

        if(month == null){

            logger.warn(csvReportLine(regNumber, "Unknown month name", monthName));

            return null;

        } else {

            return month.toString();

        }

    }

    private void addSpecimenTypes(BotanicalName taxonName, FieldUnit fieldUnit, String typeStr, TypesName typeName, boolean multiple, String regNumber){

        if(StringUtils.isEmpty(typeStr)){

            return;

        }

        typeStr = typeStr.trim().replaceAll("\\.$", "");

        Collection collection = null;

        DerivedUnit specimen = null;

        List<DerivedUnit> specimens = new ArrayList<>();

        if(multiple){

            String[] tokens = typeStr.split("\\s?,\\s?");

            for (String t : tokens) {

                // command to  list all complex parsabel types:

                // csvcut -t -c RegistrationNo_Pk,Type iapt.csv | csvgrep -c Type -m "Holotype" | egrep -o 'Holotype:\s([A-Z]*\s)[^.]*?'

                // csvcut -t -c RegistrationNo_Pk,Type iapt.csv | csvgrep -c Type -m "Holotype" | egrep -o 'Isotype[^:]*:\s([A-Z]*\s)[^.]*?'

                if(!t.isEmpty()){

                    // trying to parse the string

                    specimen = parseSpecimenType(fieldUnit, typeName, collection, t, regNumber);

                    if(specimen != null){

                        specimens.add(specimen);

                    } else {

                        // parsing was not successful make simple specimen

                        specimens.add(makeSpecimenType(fieldUnit, t));

                    }

                }

            }

        } else {

            specimen = parseSpecimenType(fieldUnit, typeName, collection, typeStr, regNumber);

            if(specimen != null) {

                specimens.add(specimen);

                // remember current collection

                collection = specimen.getCollection();

            } else {

                // parsing was not successful make simple specimen

                specimens.add(makeSpecimenType(fieldUnit, typeStr));

            }

        }

        for(DerivedUnit s : specimens){

            taxonName.addSpecimenTypeDesignation(s, typeName.status(), null, null, null, false, true);

       }

    }

    private DerivedUnit makeSpecimenType(FieldUnit fieldUnit, String titleCache) {

        DerivedUnit specimen;DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(SpecimenOrObservationType.PreservedSpecimen, fieldUnit);

        facade.setTitleCache(titleCache.trim(), true);

        specimen = facade.innerDerivedUnit();

        return specimen;

    }

    /**

     *

     * @param fieldUnit

     * @param typeName

     * @param collection

     * @param text

     * @param regNumber

     * @return

     */

    private DerivedUnit parseSpecimenType(FieldUnit fieldUnit, TypesName typeName, Collection collection, String text, String regNumber) {

        DerivedUnit specimen = null;

        String collectionCode = null;

        String subCollectionStr = null;

        String instituteStr = null;

        String accessionNumber = null;

        boolean unusualAccessionNumber = false;

        text = text.trim();

        // 1.  For Isotypes often the accession number is noted alone if the

        //     preceeding entry has a collection code.

        if(typeName .equals(TypesName.isotype) && collection != null){

            Matcher m = accessionNumberOnlyPattern.matcher(text);

            if(m.matches()){

                try {

                    accessionNumber = m.group("accNumber");

                    specimen = makeSpecimenType(fieldUnit, collection, accessionNumber);

                } catch (IllegalArgumentException e){

                    // match group acc_number not found

                }

            }

        }

        //2. try it the 'normal' way

        if(specimen == null) {

            for (Pattern p : specimenTypePatterns) {

                Matcher m = p.matcher(text);

                if (m.matches()) {

                    // collection code is mandatory

                    try {

                        collectionCode = m.group("colCode");

                    } catch (IllegalArgumentException e){

                        logger.warn(csvReportLine(regNumber, "match group colCode not found"));

                        continue;

                    }

                    try {

                        subCollectionStr = m.group("subCollection");

                    } catch (IllegalArgumentException e){

                        // match group subCollection not found

                    }

                    try {

                        instituteStr = m.group("institute");

                    } catch (IllegalArgumentException e){

                        // match group col_name not found

                    }

                    try {

                        accessionNumber = m.group("accNumber");

                        // try to improve the accessionNumber

                        if(accessionNumber!= null) {

                            accessionNumber = accessionNumber.trim();

                            Matcher m2 = accessionNumberOnlyPattern.matcher(accessionNumber);

                            String betterAccessionNumber = null;

                            if (m2.matches()) {

                                try {

                                    betterAccessionNumber = m.group("accNumber");

                                } catch (IllegalArgumentException e) {

                                    // match group acc_number not found

                                }

                            }

                            if (betterAccessionNumber != null) {

                                accessionNumber = betterAccessionNumber;

                            } else {

                                unusualAccessionNumber = true;

                            }

                        }

                    } catch (IllegalArgumentException e){

                        // match group acc_number not found

                    }

                    collection = getCollection(collectionCode, instituteStr, subCollectionStr);

                    specimen = makeSpecimenType(fieldUnit, collection, accessionNumber);

                    break;

                }

            }

        }

        if(specimen == null) {

            logger.warn(csvReportLine(regNumber, "Could not parse specimen type", typeName.name().toString(), text));

        }

        if(unusualAccessionNumber){

            logger.warn(csvReportLine(regNumber, "Unusual accession number", typeName.name().toString(), text, accessionNumber));

        }

        return specimen;

    }

    private DerivedUnit makeSpecimenType(FieldUnit fieldUnit, Collection collection, String accessionNumber) {

        DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(SpecimenOrObservationType.PreservedSpecimen, fieldUnit);

        facade.setCollection(collection);

        facade.setAccessionNumber(accessionNumber);

        return facade.innerDerivedUnit();

    }

    private BotanicalName makeBotanicalName(SimpleExcelTaxonImportState<CONFIG> state, String regNumber, String titleCacheStr, String nameStr,

                                            String authorStr, String nomRefTitle) {

        BotanicalName taxonName;// cache field for the taxonName.titleCache

        String taxonNameTitleCache = null;

        Map<String, AnnotationType> nameAnnotations = new HashMap<>();

        // TitleCache preprocessing

        if(titleCacheStr.endsWith(ANNOTATION_MARKER_STRING) || (authorStr != null && authorStr.endsWith(ANNOTATION_MARKER_STRING))){

            nameAnnotations.put("Author abbreviation not checked.", AnnotationType.EDITORIAL());

            titleCacheStr = titleCacheStr.replace(ANNOTATION_MARKER_STRING, "").trim();

            authorStr = authorStr.replace(ANNOTATION_MARKER_STRING, "").trim();

        }

        // parse the full taxon name

        if(!StringUtils.isEmpty(nomRefTitle)){

            String referenceSeparator = nomRefTitle.startsWith("in ") ? " " : ", ";

            String taxonFullNameStr = titleCacheStr + referenceSeparator + nomRefTitle;

            logger.debug(":::::" + taxonFullNameStr);

            taxonName = (BotanicalName) nameParser.parseReferencedName(taxonFullNameStr, NomenclaturalCode.ICNAFP, null);

        } else {

            taxonName = (BotanicalName) nameParser.parseFullName(titleCacheStr, NomenclaturalCode.ICNAFP, null);

        }

        taxonNameTitleCache = taxonName.getTitleCache().trim();

        if (taxonName.isProtectedTitleCache()) {

            logger.warn(csvReportLine(regNumber, "Name could not be parsed", titleCacheStr));

        } else {

            boolean doRestoreTitleCacheStr = false;

            // Check if titleCache and nameCache are plausible

            String titleCacheCompareStr = titleCacheStr;

            String nameCache = taxonName.getNameCache();

            String nameCompareStr = nameStr;

            if(taxonName.isBinomHybrid()){

                titleCacheCompareStr = titleCacheCompareStr.replace(" x ", " ×");

                nameCompareStr = nameCompareStr.replace(" x ", " ×");

            }

            if(taxonName.isMonomHybrid()){

                titleCacheCompareStr = titleCacheCompareStr.replaceAll("^X ", "× ");

                nameCompareStr = nameCompareStr.replace("^X ", "× ");

            }

            if(authorStr != null && authorStr.contains(" et ")){

                titleCacheCompareStr = titleCacheCompareStr.replaceAll(" et ", " & ");

            }

            if (!taxonNameTitleCache.equals(titleCacheCompareStr)) {

                logger.warn(csvReportLine(regNumber, "The generated titleCache differs from the imported string", taxonNameTitleCache, " != ", titleCacheStr, " ==> original titleCacheStr has been restored"));

                doRestoreTitleCacheStr = true;

            }

            if (!nameCache.trim().equals(nameCompareStr)) {

                logger.warn(csvReportLine(regNumber, "The parsed nameCache differs from field '" + NAMESTRING + "'", nameCache, " != ", nameCompareStr));

            }

            //  Author

            //nameParser.handleAuthors(taxonName, titleCacheStr, authorStr);

            //if (!titleCacheStr.equals(taxonName.getTitleCache())) {

            //    logger.warn(regNumber + ": titleCache has changed after setting authors, will restore original titleCacheStr");

            //    doRestoreTitleCacheStr = true;

            //}

            if(doRestoreTitleCacheStr){

                taxonName.setTitleCache(titleCacheStr, true);

            }

            // deduplicate

            replaceAuthorNamesAndNomRef(state, taxonName);

        }

        // Annotations

        if(!nameAnnotations.isEmpty()){

            for(String text : nameAnnotations.keySet()){

                taxonName.addAnnotation(Annotation.NewInstance(text, nameAnnotations.get(text), Language.DEFAULT()));

            }

            getNameService().save(taxonName);

        }

        return taxonName;

    }

    /**

     * @param state

     * @return

     */

    private TaxonNode getClassificationRootNode(IAPTImportState state) {

     //   Classification classification = state.getClassification();

     //   if (classification == null){

     //       IAPTImportConfigurator config = state.getConfig();

     //       classification = Classification.NewInstance(state.getConfig().getClassificationName());

     //       classification.setUuid(config.getClassificationUuid());

     //       classification.setReference(config.getSecReference());

     //       classification = getClassificationService().find(state.getConfig().getClassificationUuid());

     //   }

        TaxonNode rootNode = state.getRootNode();

        if (rootNode == null){

            rootNode = getTaxonNodeService().find(ROOT_UUID);

        }

        if (rootNode == null){

            Classification classification = state.getClassification();

            if (classification == null){

                Reference sec = state.getSecReference();

                String classificationName = state.getConfig().getClassificationName();

                Language language = Language.DEFAULT();

                classification = Classification.NewInstance(classificationName, sec, language);

                state.setClassification(classification);

                classification.setUuid(state.getConfig().getClassificationUuid());

                classification.getRootNode().setUuid(ROOT_UUID);

                getClassificationService().save(classification);

            }

            rootNode = classification.getRootNode();

            state.setRootNode(rootNode);

        }

        return rootNode;

    }

    private Collection getCollection(String collectionCode, String instituteStr, String subCollectionStr){

        Collection superCollection = null;

        if(subCollectionStr != null){

            superCollection = getCollection(collectionCode, instituteStr, null);

            collectionCode = subCollectionStr;

            instituteStr = null;

        }

        final String key = collectionCode + "-#i:" + StringUtils.defaultString(instituteStr);

        Collection collection = collectionMap.get(key);

        if(collection == null) {

            collection = Collection.NewInstance();

            collection.setCode(collectionCode);

            if(instituteStr != null){

                collection.setInstitute(Institution.NewNamedInstance(instituteStr));

            }

            if(superCollection != null){

                collection.setSuperCollection(superCollection);

            }

            collectionMap.put(key, collection);

            getCollectionService().save(collection);

        }

        return collection;

    }

    /**

     * @param record

     * @param originalKey

     * @param doUnescapeHtmlEntities

     * @return

     */

    private String getValue(HashMap<String, String> record, String originalKey, boolean doUnescapeHtmlEntities) {

        String value = record.get(originalKey);

        value = fixCharacters(value);

        if (! StringUtils.isBlank(value)) {

        	if (logger.isDebugEnabled()) {

        	    logger.debug(originalKey + ": " + value);

        	}

        	value = CdmUtils.removeDuplicateWhitespace(value.trim()).toString();

            if(doUnescapeHtmlEntities){

                value = StringEscapeUtils.unescapeHtml(value);

            }

        	return value.trim();

        }else{

        	return null;

        }

    }

    /**

     * Fixes broken characters.

     * For details see

     * http://dev.e-taxonomy.eu/redmine/issues/6035

     *

     * @param value

     * @return

     */

    private String fixCharacters(String value) {

        value = StringUtils.replace(value, "s$K", "š");

        value = StringUtils.replace(value, "n$K", "ň");

        value = StringUtils.replace(value, "e$K", "ě");

        value = StringUtils.replace(value, "r$K", "ř");

        value = StringUtils.replace(value, "c$K", "č");

        value = StringUtils.replace(value, "z$K", "ž");

        value = StringUtils.replace(value, "S>U$K", "Š");

        value = StringUtils.replace(value, "C>U$K", "Č");

        value = StringUtils.replace(value, "R>U$K", "Ř");

        value = StringUtils.replace(value, "Z>U$K", "Ž");

        value = StringUtils.replace(value, "g$K", "ǧ");

        value = StringUtils.replace(value, "s$A", "ś");

        value = StringUtils.replace(value, "n$A", "ń");

        value = StringUtils.replace(value, "c$A", "ć");

        value = StringUtils.replace(value, "e$E", "ę");

        value = StringUtils.replace(value, "o$H", "õ");

        value = StringUtils.replace(value, "s$C", "ş");

        value = StringUtils.replace(value, "t$C", "ț");

        value = StringUtils.replace(value, "S>U$C", "Ş");

        value = StringUtils.replace(value, "a$O", "å");

        value = StringUtils.replace(value, "A>U$O", "Å");