EDIT

// $Id$

/**

 * Copyright (C) 2013 EDIT

 * European Distributed Institute of Taxonomy

 * http://www.e-taxonomy.eu

 *

 * The contents of this file are subject to the Mozilla Public License Version 1.1

 * See LICENSE.TXT at the top of this package for the full license terms.

 */

package eu.etaxonomy.cdm.io.taxonx2013;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.net.URI;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Set;

import java.util.UUID;

import java.util.regex.Pattern;

import javax.xml.transform.TransformerException;

import javax.xml.transform.TransformerFactoryConfigurationError;

import org.apache.commons.lang.StringUtils;

import org.apache.log4j.Logger;

import org.w3c.dom.Node;

import org.w3c.dom.NodeList;

import com.ibm.lsid.MalformedLSIDException;

import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;

import eu.etaxonomy.cdm.api.service.pager.Pager;

import eu.etaxonomy.cdm.model.agent.AgentBase;

import eu.etaxonomy.cdm.model.agent.Person;

import eu.etaxonomy.cdm.model.common.CdmBase;

import eu.etaxonomy.cdm.model.common.IdentifiableSource;

import eu.etaxonomy.cdm.model.common.LSID;

import eu.etaxonomy.cdm.model.common.Language;

import eu.etaxonomy.cdm.model.common.OriginalSourceType;

import eu.etaxonomy.cdm.model.common.UuidAndTitleCache;

import eu.etaxonomy.cdm.model.description.Feature;

import eu.etaxonomy.cdm.model.description.FeatureNode;

import eu.etaxonomy.cdm.model.description.FeatureTree;

import eu.etaxonomy.cdm.model.description.IndividualsAssociation;

import eu.etaxonomy.cdm.model.description.TaxonDescription;

import eu.etaxonomy.cdm.model.description.TaxonNameDescription;

import eu.etaxonomy.cdm.model.description.TextData;

import eu.etaxonomy.cdm.model.name.BacterialName;

import eu.etaxonomy.cdm.model.name.BotanicalName;

import eu.etaxonomy.cdm.model.name.NomenclaturalCode;

import eu.etaxonomy.cdm.model.name.NomenclaturalStatus;

import eu.etaxonomy.cdm.model.name.NomenclaturalStatusType;

import eu.etaxonomy.cdm.model.name.NonViralName;

import eu.etaxonomy.cdm.model.name.Rank;

import eu.etaxonomy.cdm.model.name.TaxonNameBase;

import eu.etaxonomy.cdm.model.name.ZoologicalName;

import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;

import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;

import eu.etaxonomy.cdm.model.reference.Reference;

import eu.etaxonomy.cdm.model.reference.ReferenceFactory;

import eu.etaxonomy.cdm.model.taxon.Classification;

import eu.etaxonomy.cdm.model.taxon.Synonym;

import eu.etaxonomy.cdm.model.taxon.SynonymRelationshipType;

import eu.etaxonomy.cdm.model.taxon.Taxon;

import eu.etaxonomy.cdm.model.taxon.TaxonBase;

import eu.etaxonomy.cdm.model.taxon.TaxonNode;

import eu.etaxonomy.cdm.persistence.query.MatchMode;

import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;

import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;

import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;

/**

 * @author pkelbert

 * @date 2 avr. 2013

 *

 */

public class TaxonXTreatmentExtractor extends TaxonXExtractor{

    private static final Logger logger = Logger.getLogger(TaxonXTreatmentExtractor.class);

    private static final String notMarkedUp = "Not marked-up";

    private static final UUID proIbioTreeUUID = UUID.fromString("2c49f506-c7f7-44de-a8b9-2e695de3769c");

    private static final UUID OtherUUID = UUID.fromString("6465f8aa-2175-446f-807e-7163994b120f");

    private static final UUID NotMarkedUpUUID = UUID.fromString("796fe3a5-2c9c-4a89-b298-7598ca944063");

    private static final boolean skippQuestion = true;

    private final NomenclaturalCode nomenclaturalCode;

    private Classification classification;

    private  String treatmentMainName,originalTreatmentName;

    private final HashMap<String,Map<String,String>> namesMap = new HashMap<String, Map<String,String>>();

    private final Pattern keypattern = Pattern.compile("^(\\d+.*|-\\d+.*)");

    private final Pattern keypatternend = Pattern.compile("^.+?\\d$");

    private boolean maxRankRespected =false;

    private Map<String, Feature> featuresMap;

    private MyName currentMyName;

    private Reference<?> sourceUrlRef;

    private final TaxonXAddSources sourceHandler = new TaxonXAddSources();

    /**

     * @param nomenclaturalCode

     * @param classification

     * @param importer

     * @param configState

     */

    public TaxonXTreatmentExtractor(NomenclaturalCode nomenclaturalCode, Classification classification, TaxonXImport importer,

            TaxonXImportState configState,Map<String, Feature> featuresMap,  Reference<?> urlSource ) {

        this.nomenclaturalCode=nomenclaturalCode;

        this.classification = classification;

        this.importer=importer;

        this.configState=configState;

        this.featuresMap=featuresMap;

        this.sourceUrlRef =urlSource;

        prepareCollectors(configState, importer.getAgentService());

        this.sourceHandler.setSourceUrlRef(sourceUrlRef);

        this.sourceHandler.setImporter(importer);

        this.sourceHandler.setConfigState(configState);

    }

    /**

     * extracts all the treament information and save them

     * @param treatmentnode: the XML Node

     * @param tosave: the list of object to save into the CDM

     * @param refMods: the reference extracted from the MODS

     * @param sourceName: the URI of the document

     */

    @SuppressWarnings({ "rawtypes", "unused" })

    protected void extractTreatment(Node treatmentnode, List<Object> tosave, Reference<?> refMods, URI sourceName) {

        logger.info("extractTreatment");

        List<TaxonNameBase> nametosave = new ArrayList<TaxonNameBase>();

        NodeList children = treatmentnode.getChildNodes();

        Taxon acceptedTaxon =null;

        Taxon defaultTaxon =null;

        boolean refgroup=false;

        for (int i=0;i<children.getLength();i++){

            if(children.item(i).getNodeName().equalsIgnoreCase("tax:ref_group")) {

                refgroup=true;

            }

        }

        for (int i=0;i<children.getLength();i++){

            if (children.item(i).getNodeName().equalsIgnoreCase("tax:nomenclature")){

                NodeList nomenclature = children.item(i).getChildNodes();

                boolean containsName=false;

                for(int k=0;k<nomenclature.getLength();k++){

                    if(nomenclature.item(k).getNodeName().equalsIgnoreCase("tax:name")){

                        containsName=true;

                        break;

                    }

                }

                if (containsName){

                    reloadClassification();

                    //extract "main" the scientific name

                    try{

                        acceptedTaxon = extractNomenclature(children.item(i),nametosave,refMods);

                    }catch(ClassCastException e){e.printStackTrace();System.exit(0);}

                    //                    System.out.println("acceptedTaxon : "+acceptedTaxon);

                }

            }

            else if (children.item(i).getNodeName().equalsIgnoreCase("tax:ref_group") && maxRankRespected){

                reloadClassification();

                //extract the References within the document

                extractReferences(children.item(i),nametosave,acceptedTaxon,refMods);

            }

            else if (children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("multiple") && maxRankRespected){

                File file = new File("/home/pkelbert/Bureau/multipleTaxonX.txt");

                FileWriter writer;

                try {

                    writer = new FileWriter(file ,true);

                    writer.write(sourceName+"\n");

                    writer.flush();

                    writer.close();

                } catch (IOException e1) {

                    // TODO Auto-generated catch block

                    e1.printStackTrace();

                }

                //                String multiple = askMultiple(children.item(i));

                String multiple = "Other";

                if (multiple.equalsIgnoreCase("other")) {

                    extractSpecificFeatureNotStructured(children.item(i),acceptedTaxon, defaultTaxon,nametosave, refMods,multiple);

                }

                else

                    if (multiple.equalsIgnoreCase("synonyms")) {

                        try{

                            extractSynonyms(children.item(i),acceptedTaxon, refMods);

                        }catch(NullPointerException e){

                            logger.warn("the accepted taxon is maybe null");

                        }

                    }

                    else

                        if(multiple.equalsIgnoreCase("material examined")){

                            extractMaterials(children.item(i),acceptedTaxon, refMods, nametosave);

                        }

                        else

                            if (multiple.equalsIgnoreCase("distribution")){

                                extractDistribution(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods);

                            }

                            else

                                if (multiple.equalsIgnoreCase("type status")){

                                    extractDescriptionWithReference(children.item(i),acceptedTaxon,defaultTaxon,refMods, "TypeStatus");

                                }

                                else

                                    if (multiple.equalsIgnoreCase("vernacular name")){

                                        extractDescriptionWithReference(children.item(i),acceptedTaxon,defaultTaxon,refMods, Feature.COMMON_NAME().getTitleCache());

                                    }

                                    else{

                                        extractSpecificFeature(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods,multiple);

                                    }

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("biology_ecology") && maxRankRespected){

                extractFeature(children.item(i),acceptedTaxon,defaultTaxon, nametosave, refMods, Feature.BIOLOGY_ECOLOGY());

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("vernacularName") && maxRankRespected){

                extractDescriptionWithReference(children.item(i),acceptedTaxon,defaultTaxon,refMods, Feature.COMMON_NAME().getTitleCache());

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("description") && maxRankRespected){

                extractFeature(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods, Feature.DESCRIPTION());

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("diagnosis") && maxRankRespected){

                extractFeature(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods,Feature.DIAGNOSIS());

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("discussion") && maxRankRespected){

                extractFeature(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods, Feature.DISCUSSION());

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("note") && maxRankRespected){

                extractFeature(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods, Feature.DESCRIPTION());

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("distribution") && maxRankRespected){

                extractDistribution(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods);

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("etymology") && maxRankRespected){

                extractFeature(children.item(i),acceptedTaxon,defaultTaxon,nametosave,refMods,Feature.ETYMOLOGY());

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("materials_examined") && maxRankRespected){

                extractMaterials(children.item(i),acceptedTaxon, refMods, nametosave);

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:figure") && maxRankRespected){

                extractSpecificFeature(children.item(i),acceptedTaxon,defaultTaxon, nametosave, refMods, "Figure");

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("Other") && maxRankRespected){

                extractSpecificFeature(children.item(i),acceptedTaxon,defaultTaxon, nametosave, refMods, "table");

            }

            else if(children.item(i).getNodeName().equalsIgnoreCase("tax:div") &&

                    children.item(i).getAttributes().getNamedItem("type").getNodeValue().equalsIgnoreCase("key") && maxRankRespected){

                //TODO IGNORE keys for the moment

                //extractKey(children.item(i),acceptedTaxon, nametosave,source, refMods);

                extractSpecificFeatureNotStructured(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods,"Keys - unparsed");

            }

            else{

                if (!children.item(i).getNodeName().equalsIgnoreCase("tax:pb")){

                    //logger.info("ANOTHER KIND OF NODES: "+children.item(i).getNodeName()+", "+children.item(i).getAttributes());

                    if (children.item(i).getAttributes() !=null) {

                        //logger.info(children.item(i).getAttributes().item(0));

                    }

                    extractSpecificFeatureNotStructured(children.item(i),acceptedTaxon,defaultTaxon,nametosave, refMods,notMarkedUp);

                }

            }

        }

        //        logger.info("saveUpdateNames");

        if (maxRankRespected){

            importer.getNameService().saveOrUpdate(nametosave);

            importer.getClassificationService().saveOrUpdate(classification);

            //logger.info("saveUpdateNames-ok");

        }

        buildFeatureTree();

    }

    protected Map<String,Feature> getFeaturesUsed(){

        return featuresMap;

    }

    /**

     *

     */

    private void buildFeatureTree() {

        logger.info("buildFeatureTree");

        FeatureTree proibiospheretree = importer.getFeatureTreeService().find(proIbioTreeUUID);

        if (proibiospheretree == null){

            List<FeatureTree> trees = importer.getFeatureTreeService().list(FeatureTree.class, null, null, null, null);

            if (trees.size()==1) {

                FeatureTree ft = trees.get(0);

                if (featuresMap==null) {

                    featuresMap=new HashMap<String, Feature>();

                }

                for (Feature feature: ft.getDistinctFeatures()){

                    if(feature!=null) {

                        featuresMap.put(feature.getTitleCache(), feature);

                    }

                }

            }

            proibiospheretree = FeatureTree.NewInstance();

            proibiospheretree.setUuid(proIbioTreeUUID);

        }

        //        FeatureNode root = proibiospheretree.getRoot();

        FeatureNode root2 = proibiospheretree.getRoot();

        if (root2 != null){

            int nbChildren = root2.getChildCount()-1;

            while (nbChildren>-1){

                try{

                    root2.removeChild(nbChildren);

                }catch(Exception e){logger.warn("Can't remove child from FeatureTree "+e);}

                nbChildren --;

            }

        }

        for (Feature feature:featuresMap.values()) {

            root2.addChild(FeatureNode.NewInstance(feature));

        }

        importer.getFeatureTreeService().saveOrUpdate(proibiospheretree);

    }

    /**

     * @param keys

     * @param acceptedTaxon: the current acceptedTaxon

     * @param nametosave: the list of objects to save into the CDM

     * @param refMods: the current reference extracted from the MODS

     */

    /*   @SuppressWarnings("rawtypes")

    private void extractKey(Node keys, Taxon acceptedTaxon,List<TaxonNameBase> nametosave, Reference<?> refMods) {

        acceptedTaxon = CdmBase.deproxy(acceptedTaxon, Taxon.class);

        NodeList children = keys.getChildNodes();

        String key="";

        PolytomousKey poly =  PolytomousKey.NewInstance();

        poly.addSource(OriginalSourceType.Import, null,null,refMods,null);

        poly.addSource(OriginalSourceType.Import, null,null,sourceUrlRef,null);

        poly.addTaxonomicScope(acceptedTaxon);

        poly.setTitleCache("bloup", true);

        //        poly.addCoveredTaxon(acceptedTaxon);

        PolytomousKeyNode root = poly.getRoot();

        PolytomousKeyNode previous = null,tmpKey=null;

        Taxon taxonKey=null;

        List<PolytomousKeyNode> polyNodes = new ArrayList<PolytomousKeyNode>();

        //        String fullContent = keys.getTextContent();

        for (int i=0;i<children.getLength();i++){

            if (children.item(i).getNodeName().equalsIgnoreCase("tax:p")){

                NodeList paragraph = children.item(i).getChildNodes();

                key="";

                taxonKey=null;

                for (int j=0;j<paragraph.getLength();j++){

                    if (paragraph.item(j).getNodeName().equalsIgnoreCase("#text")){

                        if (! paragraph.item(j).getTextContent().trim().isEmpty()){

                            key+=paragraph.item(j).getTextContent().trim();

                            //                            logger.info("KEY: "+j+"--"+key);

                        }

                    }

                    if(paragraph.item(j).getNodeName().equalsIgnoreCase("tax:name")){

                        taxonKey=getTaxonFromXML(paragraph.item(j),nametosave,refMods);

                    }

                }

                //                logger.info("keypattern.matcher(key).matches(): "+keypattern.matcher(key).matches());

                if (keypattern.matcher(key).matches()){

                    tmpKey = PolytomousKeyNode.NewInstance(key);

                    if (taxonKey!=null) {

                        tmpKey.setTaxon(taxonKey);

                    }

                    polyNodes.add(tmpKey);

                    if (previous == null) {

                        root.addChild(tmpKey);

                    } else {

                        previous.addChild(tmpKey);

                    }

                }else{

                    if (!key.isEmpty()){

                        tmpKey=PolytomousKeyNode.NewInstance(key);

                        if (taxonKey!=null) {

                            tmpKey.setTaxon(taxonKey);

                        }

                        polyNodes.add(tmpKey);

                        if (keypatternend.matcher(key).matches()) {

                            root.addChild(tmpKey);

                            previous=tmpKey;

                        } else{

                            previous.addChild(tmpKey);

                        }

                    }

                }

            }

        }

        importer.getPolytomousKeyNodeService().saveOrUpdate(polyNodes);

        importer.getPolytomousKeyService().saveOrUpdate(poly);

    }

     */

    //    /**

    //     * @param taxons: the XML Nodegroup

    //     * @param nametosave: the list of objects to save into the CDM

    //     * @param acceptedTaxon: the current accepted Taxon

    //     * @param refMods: the current reference extracted from the MODS

    //     *

    //     * @return Taxon object built

    //     */

    //    @SuppressWarnings({ "rawtypes", "unchecked" })

    //    private Taxon getTaxonFromXML(Node taxons, List<TaxonNameBase> nametosave, Reference<?> refMods) {

    //        //        logger.info("getTaxonFromXML");

    //        //        logger.info("acceptedTaxon: "+acceptedTaxon);

    //

    //        // TaxonNameBase nameToBeFilled = null;

    //

    //        currentMyName = new MyName();

    //        NomenclaturalStatusType statusType = null;

    //

    //        try {

    //            currentMyName = extractScientificName(taxons);

    //            if (!currentMyName.getStatus().isEmpty()){

    //                try {

    //                    statusType = nomStatusString2NomStatus(currentMyName.getStatus());

    //                } catch (UnknownCdmTypeException e) {

    //                    addProblematicStatusToFile(currentMyName.getStatus());

    //                    logger.warn("Problem with status");

    //                }

    //            }

    //

    //        } catch (TransformerFactoryConfigurationError e1) {

    //            logger.warn(e1);

    //        } catch (TransformerException e1) {

    //            logger.warn(e1);

    //        }

    //        /*  INonViralNameParser parser = NonViralNameParserImpl.NewInstance();

    //

    //        nameToBeFilled = parser.parseFullName(currentMyName.getName(), nomenclaturalCode, currentMyName.getRank());

    //        if (nameToBeFilled.hasProblem() &&

    //                !((nameToBeFilled.getParsingProblems().size()==1) && nameToBeFilled.getParsingProblems().contains(ParserProblem.CheckRank)) ) {

    //            //            if (nameToBeFilled.hasProblem() && nameToBeFilled.getParsingProblems().contains(ParserProblem.UnparsableNamePart)){

    //            addProblemNameToFile(currentMyName.getName(),"",nomenclaturalCode,currentMyName.getRank());

    //            nameToBeFilled=solveNameProblem(currentMyName.getOriginalName(), currentMyName.getName(),parser, currentMyName.getAuthor(), currentMyName.getRank());

    //        }

    //

    //        nameToBeFilled = getTaxonNameBase(nameToBeFilled,nametosave,statusType);

    //         */

    //        TaxonNameBase nameToBeFilled = currentMyName.getTaxonNameBase();

    //        Taxon t = currentMyName.getTaxon();

    //        //        importer.getNameService().saveOrUpdate(nametosave);

    //        /*    Taxon t = importer.getTaxonService().findBestMatchingTaxon(nameToBeFilled.getTitleCache());

    //         */

    //        boolean statusMatch=false;

    //        if(t !=null ){

    //            statusMatch=compareStatus(t, statusType);

    //        }

    //        if (t ==null || (t != null && !statusMatch)){

    //            if(statusType != null) {

    //                nameToBeFilled.addStatus(NomenclaturalStatus.NewInstance(statusType));

    //            }

    //            t= new Taxon(nameToBeFilled,(Reference<?>) nameToBeFilled.getNomenclaturalReference() );//TODO TOFIX reference

    //            if (t.getSec() == null) {

    //                t.setSec(refMods);

    //            }

    //            if(!configState.getConfig().doKeepOriginalSecundum()) {

    //                t.setSec(configState.getConfig().getSecundum());

    //                logger.info("SET SECUNDUM "+configState.getConfig().getSecundum());

    //            }

    //            t.addSource(OriginalSourceType.Import,null,null,refMods,null);

    //            t.addSource(OriginalSourceType.Import, null,null,sourceUrlRef,null);

    //

    //

    //            if (!currentMyName.getIdentifier().isEmpty() && (currentMyName.getIdentifier().length()>2)){

    //                setLSID(currentMyName.getIdentifier(), t);

    //            }

    //

    //            //            Taxon parentTaxon = currentMyName.getHigherTaxa();

    //            //            if (parentTaxon == null && !skippQuestion) {

    //            //                parentTaxon =  askParent(t, classification);

    //            //            }

    //            //            if (parentTaxon ==null){

    //            //                while (parentTaxon == null) {

    //            //                    System.out.println("parent is null");

    //            //                    parentTaxon = createParent(t, refMods);

    //            //                    classification.addParentChild(parentTaxon, t, refMods, null);

    //            //                }

    //            //            }else{

    //            //                classification.addParentChild(parentTaxon, t, refMods, null);

    //            //            }

    //        }

    //        else{

    //            t = CdmBase.deproxy(t, Taxon.class);

    //        }

    //        if (!configState.getConfig().doKeepOriginalSecundum()) {

    //            t.setSec(configState.getConfig().getSecundum());

    //            logger.info("SET SECUNDUM "+configState.getConfig().getSecundum());

    //        }

    //        return t;

    //    }

    //    private Taxon getTaxonFromTaxonNameBase(TaxonNameBase tnb,Reference<?> ref){

    //        Taxon taxon = null;

    ////        System.out.println(tnb.getTitleCache());

    //        Taxon cc= importer.getTaxonService().findBestMatchingTaxon(tnb.getTitleCache());

    //        if (cc != null){

    //            if ((cc.getSec() == null || cc.getSec().toString().isEmpty()) || (cc.getSec() != null &&

    //                    cc.getSec().getTitleCache().equalsIgnoreCase(ref.getTitleCache()))) {

    //                if(cc.getSec() == null || cc.getSec().toString().isEmpty()){

    //                    cc.setSec(ref);

    //                    importer.getTaxonService().saveOrUpdate(cc);

    //                }

    //                taxon=cc;

    //            }

    //        }

    //        else{

    //            //            List<TaxonBase> c = importer.getTaxonService().searchTaxaByName(tnb.getTitleCache(), ref);

    //            List<TaxonBase> c = importer.getTaxonService().list(TaxonBase.class, 0, 0, null, null);

    //            for (TaxonBase b : c) {

    //                try{

    //                    taxon = (Taxon) b;

    //                }catch(ClassCastException e){logger.warn("error while casting existing taxonnamebase");}

    //            }

    //        }

    //        if (taxon == null){

    ////            System.out.println("NEW TAXON HERE "+tnb.toString()+", "+ref.toString());

    //            taxon = Taxon.NewInstance(tnb, ref); //sec set null

    //            importer.getTaxonService().save(taxon);

    //

    //        }

    //        taxon = (Taxon) importer.getTaxonService().find(taxon.getUuid());

    //

    //        boolean exist = false;

    //        for (TaxonNode p : classification.getAllNodes()){

    //            if(p.getTaxon().equals(taxon)) {

    //                exist =true;

    //            }

    //        }

    //        if (!exist){

    //            taxon = (Taxon) importer.getTaxonService().find(taxon.getUuid());

    //            Taxon parentTaxon = currentMyName.getHigherTaxa();

    //            if (parentTaxon != null) {

    //                classification.addParentChild(parentTaxon, taxon, ref, null);

    //            } else {

    //                System.out.println("HERE???");

    //                classification.addChildTaxon(taxon, ref, null);

    //            }

    //            importer.getClassificationService().saveOrUpdate(classification);

    //            //                        refreshTransaction();

    //        }

    //        taxon = CdmBase.deproxy(taxon, Taxon.class);

    //        //        System.out.println("TAXON RETOURNE : "+taxon.getTitleCache());

    //        return taxon;

    //    }

    /**

     * @param taxons: the XML Nodegroup

     * @param nametosave: the list of objects to save into the CDM

     * @param acceptedTaxon: the current accepted Taxon

     * @param refMods: the current reference extracted from the MODS

     *

     * @return Taxon object built

     */

    @SuppressWarnings({ "rawtypes", "unused" })

    private TaxonNameBase getTaxonNameBaseFromXML(Node taxons, List<TaxonNameBase> nametosave, Reference<?> refMods, boolean isSynonym) {

        //        logger.info("getTaxonFromXML");

        //        logger.info("acceptedTaxon: "+acceptedTaxon);

        logger.info("getTaxonNameBaseFromXML");

        TaxonNameBase nameToBeFilled = null;

        currentMyName=new MyName(isSynonym);

        NomenclaturalStatusType statusType = null;

        try {

            currentMyName = extractScientificName(taxons,refMods);

        } catch (TransformerFactoryConfigurationError e1) {

            logger.warn(e1);

        } catch (TransformerException e1) {

            logger.warn(e1);

        }

        /* INonViralNameParser parser = NonViralNameParserImpl.NewInstance();

        nameToBeFilled = parser.parseFullName(currentMyName.getName(), nomenclaturalCode, currentMyName.getRank());

        if (nameToBeFilled.hasProblem() &&

                !((nameToBeFilled.getParsingProblems().size()==1) && nameToBeFilled.getParsingProblems().contains(ParserProblem.CheckRank)) ) {

            //            if (nameToBeFilled.hasProblem() && nameToBeFilled.getParsingProblems().contains(ParserProblem.UnparsableNamePart)){

            addProblemNameToFile(currentMyName.getName(),"",nomenclaturalCode,currentMyName.getRank());

            nameToBeFilled=solveNameProblem(currentMyName.getOriginalName(), currentMyName.getName(),parser,currentMyName.getAuthor(), currentMyName.getRank());

        }

        nameToBeFilled = getTaxonNameBase(nameToBeFilled,nametosave,statusType);

         */

        nameToBeFilled = currentMyName.getTaxonNameBase();

        return nameToBeFilled;

    }

    //    @SuppressWarnings("rawtypes")

    //    private TaxonNameBase getTaxonNameBase (TaxonNameBase name, List<TaxonNameBase> nametosave, NomenclaturalStatusType statusType){

    //        List<TaxonNameBase> names = importer.getNameService().list(TaxonNameBase.class, null, null, null, null);

    //        for (TaxonNameBase tb : names){

    //            if (tb.getTitleCache().equalsIgnoreCase(name.getTitleCache())) {

    //                boolean statusMatch=false;

    //                if(tb !=null ){

    //                    statusMatch=compareStatus(tb, statusType);

    //                }

    //                if (!statusMatch){

    //                    if(statusType != null) {

    //                        name.addStatus(NomenclaturalStatus.NewInstance(statusType));

    //                    }

    //                }else

    //                {

    //                    logger.info("TaxonNameBase FOUND"+name.getTitleCache());

    //                    return  CdmBase.deproxy(tb, TaxonNameBase.class);

    //                }

    //            }

    //        }

    //        //        logger.info("TaxonNameBase NOT FOUND "+name.getTitleCache());

    //        //        System.out.println("add name "+name);

    //        nametosave.add(name);

    //        name = CdmBase.deproxy(name, TaxonNameBase.class);

    //        return name;

    //

    //    }

    //    /**

    //     * @param tb

    //     * @param statusType

    //     * @return

    //     */

    //    private boolean compareStatus(TaxonNameBase tb, NomenclaturalStatusType statusType) {

    //        boolean statusMatch=false;

    //        //found one taxon

    //        Set<NomenclaturalStatus> status = tb.getStatus();

    //        if (statusType!=null && status.size()>0){ //the statusType is known for both taxon

    //            for (NomenclaturalStatus st:status){

    //                NomenclaturalStatusType stype = st.getType();

    //                if (stype.toString().equalsIgnoreCase(statusType.toString())) {

    //                    statusMatch=true;

    //                }

    //            }

    //        }

    //        else{

    //            if(statusType == null && status.size()==0) {//there is no statusType, we can assume it's the same

    //                statusMatch=true;

    //            }

    //        }

    //        return statusMatch;

    //    }

    /**

     *

     */

    private void reloadClassification() {

        logger.info("reloadClassification");

        Classification cl = importer.getClassificationService().find(classification.getUuid());

        if (cl != null){

            classification=cl;

        }else{

            importer.getClassificationService().saveOrUpdate(classification);

            classification = importer.getClassificationService().find(classification.getUuid());

        }

    }

    //    /**

    //     * Create a Taxon for the current NameBase, based on the current reference

    //     * @param taxonNameBase

    //     * @param refMods: the current reference extracted from the MODS

    //     * @return Taxon

    //     */

    //    @SuppressWarnings({ "unused", "rawtypes" })

    //    private Taxon getTaxon(TaxonNameBase taxonNameBase, Reference<?> refMods) {

    //        Taxon t = new Taxon(taxonNameBase,null );

    //        if (!configState.getConfig().doKeepOriginalSecundum() || (t.getSec() == null)) {

    //            t.setSec(configState.getConfig().getSecundum());

    //            logger.info("SET SECUNDUM "+configState.getConfig().getSecundum());

    //        }

    //        /*<<<<<<< .courant

    //        boolean sourceExists=false;

    //        Set<IdentifiableSource> sources = t.getSources();

    //        for (IdentifiableSource src : sources){

    //            String micro = src.getCitationMicroReference();

    //            Reference r = src.getCitation();

    //            if (r.equals(refMods) && micro == null) {

    //                sourceExists=true;

    //            }

    //        }

    //        if(!sourceExists) {

    //            t.addSource(null,null,refMods,null);

    //        }

    //=======*/

    //        t.addSource(OriginalSourceType.Import,null,null,refMods,null);

    //        t.addSource(OriginalSourceType.Import, null,null,sourceUrlRef,null);

    //        return t;

    //    }

    private void  extractDescriptionWithReference(Node typestatus, Taxon acceptedTaxon, Taxon defaultTaxon, Reference<?> refMods,

            String featureName) {

        //        System.out.println("extractDescriptionWithReference !");

        logger.info("extractDescriptionWithReference");

        NodeList children = typestatus.getChildNodes();

        Feature currentFeature=getFeatureObjectFromString(featureName);

        String r="";String s="";

        for (int i=0;i<children.getLength();i++){

            if (children.item(i).getNodeName().equalsIgnoreCase("tax:p")){

                s+=children.item(i).getTextContent().trim();

            }

            if (children.item(i).getNodeName().equalsIgnoreCase("tax:bibref")){

                r+= children.item(i).getTextContent().trim();

            }

            if (s.indexOf(r)>-1) {

                s=s.split(r)[0];

            }

        }

        Reference<?> currentref =  ReferenceFactory.newGeneric();

        if(!r.isEmpty()) {

            currentref.setTitleCache(r, true);

        } else {

            currentref=refMods;

        }

        setParticularDescription(s,acceptedTaxon,defaultTaxon, currentref, refMods,currentFeature);

    }

    /**

     * @param nametosave

     * @param distribution: the XML node group

     * @param acceptedTaxon: the current accepted Taxon

     * @param defaultTaxon: the current defaultTaxon, only used if there is no accepted name

     * @param refMods: the current reference extracted from the MODS

     */

    @SuppressWarnings("rawtypes")

    private void extractDistribution(Node distribution, Taxon acceptedTaxon, Taxon defaultTaxon, List<TaxonNameBase> nametosave, Reference<?> refMods) {

        logger.info("extractDistribution");

        //        logger.info("acceptedTaxon: "+acceptedTaxon);

        NodeList children = distribution.getChildNodes();

        Map<Integer,List<MySpecimenOrObservation>> specimenOrObservations = new HashMap<Integer, List<MySpecimenOrObservation>>();

        Map<Integer,String> descriptionsFulltext = new HashMap<Integer,String>();

        for (int i=0;i<children.getLength();i++){

            if (children.item(i).getNodeName().equalsIgnoreCase("tax:p")){

                NodeList paragraph = children.item(i).getChildNodes();

                for (int j=0;j<paragraph.getLength();j++){

                    if (paragraph.item(j).getNodeName().equalsIgnoreCase("#text")){

                        extractText(descriptionsFulltext, i, paragraph.item(j));

                    }

                    else if (paragraph.item(j).getNodeName().equalsIgnoreCase("tax:name")){

                        extractInLine(nametosave, refMods, descriptionsFulltext, i,paragraph.item(j));

                    }

                    else if (paragraph.item(j).getNodeName().equalsIgnoreCase("tax:collection_event")){

                        MySpecimenOrObservation specimenOrObservation = new MySpecimenOrObservation();

                        DerivedUnit derivedUnitBase = null;

                        specimenOrObservation = extractSpecimenOrObservation(paragraph.item(j), derivedUnitBase, SpecimenOrObservationType.DerivedUnit);

                        extractTextFromSpecimenOrObservation(specimenOrObservations, descriptionsFulltext, i, specimenOrObservation);

                    }

                }

            }

        }

        int m=0;

        for (int k:descriptionsFulltext.keySet()) {

            if (k>m) {

                m=k;

            }

        }

        for (int k:specimenOrObservations.keySet()) {

            if (k>m) {

                m=k;

            }

        }

        if(acceptedTaxon!=null){

            TaxonDescription td =importer.getTaxonDescription(acceptedTaxon, false, true);

            Feature currentFeature = Feature.DISTRIBUTION();

            //        DerivedUnit derivedUnitBase=null;

            //        String descr="";

            for (int k=0;k<=m;k++){

                if(specimenOrObservations.keySet().contains(k)){

                    for (MySpecimenOrObservation soo:specimenOrObservations.get(k) ) {

                        handleAssociation(acceptedTaxon, refMods, td, soo);

                    }

                }

                if (descriptionsFulltext.keySet().contains(k)){

                    if (!stringIsEmpty(descriptionsFulltext.get(k).trim()) && (descriptionsFulltext.get(k).startsWith("Hab.") || descriptionsFulltext.get(k).startsWith("Habitat"))){

                        setParticularDescription(descriptionsFulltext.get(k),acceptedTaxon,defaultTaxon, refMods, Feature.HABITAT());

                        break;

                    }

                    else{

                        handleTextData(refMods, descriptionsFulltext, td, currentFeature, k);

                    }

                }

                if (descriptionsFulltext.keySet().contains(k) || specimenOrObservations.keySet().contains(k)){

                    acceptedTaxon.addDescription(td);

                    sourceHandler.addAndSaveSource(refMods, td, null);

                    importer.getTaxonService().saveOrUpdate(acceptedTaxon);

                }

            }

        }

    }

    /**

     * @param refMods

     * @param descriptionsFulltext

     * @param td

     * @param currentFeature

     * @param k

     */

    private void handleTextData(Reference<?> refMods, Map<Integer, String> descriptionsFulltext, TaxonDescription td,

            Feature currentFeature, int k) {

        //logger.info("handleTextData");

        TextData textData = TextData.NewInstance();

        textData.setFeature(currentFeature);

        textData.putText(Language.UNKNOWN_LANGUAGE(), descriptionsFulltext.get(k));

        sourceHandler.addSource(refMods, textData);

        td.addElement(textData);

    }

    /**

     * @param acceptedTaxon

     * @param refMods

     * @param td

     * @param soo

     */

    private void handleAssociation(Taxon acceptedTaxon, Reference<?> refMods, TaxonDescription td, MySpecimenOrObservation soo) {

        logger.info("handleAssociation");

        String descr=soo.getDescr();

        DerivedUnit derivedUnitBase = soo.getDerivedUnitBase();

        sourceHandler.addAndSaveSource(refMods, derivedUnitBase);

        TaxonDescription taxonDescription = importer.getTaxonDescription(acceptedTaxon, false, true);

        Feature feature=null;

        feature = makeFeature(derivedUnitBase);

        if(!StringUtils.isEmpty(descr)) {

            derivedUnitBase.setTitleCache(descr, true);

        }

        IndividualsAssociation indAssociation = createIndividualAssociation(refMods, derivedUnitBase, feature);

        taxonDescription.addElement(indAssociation);