cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.InputStream;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import javax.xml.parsers.DocumentBuilder;
  24 import javax.xml.parsers.DocumentBuilderFactory;
  25
  26 import org.apache.commons.lang.StringUtils;
  27 import org.apache.log4j.Logger;
  28 import org.springframework.stereotype.Component;
  29 import org.w3c.dom.Document;
  30 import org.w3c.dom.Element;
  31 import org.w3c.dom.NodeList;
  32
  33 import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
  34 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  35 import eu.etaxonomy.cdm.common.UriUtils;
  36 import eu.etaxonomy.cdm.io.specimen.SpecimenImportBase;
  37 import eu.etaxonomy.cdm.io.specimen.SpecimenUserInteraction;
  38 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  39 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  40 import eu.etaxonomy.cdm.model.agent.AgentBase;
  41 import eu.etaxonomy.cdm.model.agent.Institution;
  42 import eu.etaxonomy.cdm.model.agent.Person;
  43 import eu.etaxonomy.cdm.model.agent.Team;
  44 import eu.etaxonomy.cdm.model.common.CdmBase;
  45 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  46 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  47 import eu.etaxonomy.cdm.model.common.Language;
  48 import eu.etaxonomy.cdm.model.common.LanguageString;
  49 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  50 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  51 import eu.etaxonomy.cdm.model.common.UuidAndTitleCache;
  52 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  53 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  54 import eu.etaxonomy.cdm.model.description.Feature;
  55 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  56 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  57 import eu.etaxonomy.cdm.model.location.NamedArea;
  58 import eu.etaxonomy.cdm.model.media.Media;
  59 import eu.etaxonomy.cdm.model.name.BacterialName;
  60 import eu.etaxonomy.cdm.model.name.BotanicalName;
  61 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  62 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  63 import eu.etaxonomy.cdm.model.name.NonViralName;
  64 import eu.etaxonomy.cdm.model.name.Rank;
  65 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  66 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  67 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  68 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  69 import eu.etaxonomy.cdm.model.occurrence.Collection;
  70 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  71 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  72 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  73 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  74 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  75 import eu.etaxonomy.cdm.model.reference.Reference;
  76 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  77 import eu.etaxonomy.cdm.model.taxon.Classification;
  78 import eu.etaxonomy.cdm.model.taxon.Taxon;
  79 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  80 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  81 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  82 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  83
  84 /**
  85  * @author p.kelbert
  86  * @created 20.10.2008
  87  */
  88 @Component
  89 public class Abcd206Import extends SpecimenImportBase<Abcd206ImportConfigurator, Abcd206ImportState> {
  90     private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  91
  92     private final boolean DEBUG = true;
  93
  94     private static final String PREFERRED = "_preferred_";
  95     private static final String CODE = "_code_";
  96     private static final String COLON = ":";
  97     private static final String SPLITTER = "--";
  98     private static String prefix = "";
  99
 100     //TODO make all fields ABCD206ImportState variables
 101     private Classification classification = null;
 102     private Reference<?> ref = null;
 103
 104     private Abcd206DataHolder dataHolder;
 105     private DerivedUnit derivedUnitBase;
 106
 107     private List<OriginalSourceBase<?>> associationRefs = new ArrayList<OriginalSourceBase<?>>();
 108     boolean associationSourcesSet=false;
 109     private List<OriginalSourceBase<?>> descriptionRefs = new ArrayList<OriginalSourceBase<?>>();
 110     boolean descriptionSourcesSet=false;
 111     private List<OriginalSourceBase<?>> derivedUnitSources = new ArrayList<OriginalSourceBase<?>>();
 112     boolean derivedUnitSourcesSet=false;
 113     private boolean descriptionGroupSet = false;
 114     private TaxonDescription descriptionGroup = null;
 115
 116     public Abcd206Import() {
 117         super();
 118     }
 119
 120     /**
 121      * TODO: quick and dirty fix to avoid NonUniqueObjectExceptions
 122      * They occurred because this import is a bean therefore a singleton but uses
 123      * class variables which should be unique for every single import.
 124      */
 125     private void resetFields(){
 126         classification = null;
 127         ref = null;
 128
 129         dataHolder = null;
 130         derivedUnitBase = null;
 131
 132         associationRefs = new ArrayList<OriginalSourceBase<?>>();
 133         associationSourcesSet=false;
 134         descriptionRefs = new ArrayList<OriginalSourceBase<?>>();
 135         descriptionSourcesSet=false;
 136         derivedUnitSources = new ArrayList<OriginalSourceBase<?>>();
 137         derivedUnitSourcesSet=false;
 138         descriptionGroupSet = false;
 139         descriptionGroup = null;
 140     }
 141
 142     @Override
 143     protected boolean doCheck(Abcd206ImportState state) {
 144         logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
 145         return true;
 146     }
 147
 148
 149     @Override
 150     @SuppressWarnings("rawtypes")
 151     public void doInvoke(Abcd206ImportState state) {
 152         resetFields();
 153
 154         state.setTx(startTransaction());
 155         logger.info("INVOKE Specimen Import from ABCD2.06 XML ");
 156
 157         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 158
 159         List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
 160
 161         if (state.getConfig().isInteractWithUser()){
 162             Map<String,Reference> refMap = new HashMap<String, Reference>();
 163             for (Reference reference : references) {
 164                 if (! StringUtils.isBlank(reference.getTitleCache())) {
 165                     refMap.put(reference.getTitleCache(),reference);
 166                 }
 167             }
 168             ref = sui.askForReference(refMap);
 169
 170             if (ref == null){
 171                 String cla = sui.createNewReference();
 172                 if (refMap.get(cla)!= null) {
 173                     ref = refMap.get(cla);
 174                 } else {
 175                     ref = ReferenceFactory.newGeneric();
 176                     ref.setTitle(cla);
 177                 }
 178             }
 179             else{
 180                 ref = getReferenceService().find(ref.getUuid());
 181             }
 182         }else{
 183             if (ref==null){
 184                 String name = NB(state.getConfig().getSourceReferenceTitle());
 185                 for (Reference reference : references) {
 186                     if (! StringUtils.isBlank(reference.getTitleCache())) {
 187                         if (reference.getTitleCache().equalsIgnoreCase(name)) {
 188                             ref=reference;
 189 //                            System.out.println("FIND SAME REFERENCE");
 190                         }
 191                     }
 192                 }
 193                 if (ref == null){
 194                     ref = ReferenceFactory.newGeneric();
 195                     ref.setTitle("ABCD classic");
 196                 }
 197             }
 198         }
 199         save(ref, state);
 200         state.getConfig().setSourceReference(ref);
 201
 202         List<Classification> classificationList = getClassificationService().list(Classification.class, null, null, null, null);
 203         if (state.getConfig().isUseClassification() && state.getConfig().isInteractWithUser()){
 204             Map<String,Classification> classMap = new HashMap<String, Classification>();
 205             for (Classification tree : classificationList) {
 206                 if (! StringUtils.isBlank(tree.getTitleCache())) {
 207                     classMap.put(tree.getTitleCache(),tree);
 208                 }
 209             }
 210             classification = sui.askForClassification(classMap);
 211             if (classification == null){
 212                 String cla = sui.createNewClassification();
 213                 if (classMap.get(cla)!= null) {
 214                     classification = classMap.get(cla);
 215                 } else {
 216                     classification = Classification.NewInstance(cla, ref, Language.DEFAULT());
 217                 }
 218             }
 219             save(classification, state);
 220         }
 221         else{
 222             if (classification == null) {
 223                 String name = NB(state.getConfig().getClassificationName());
 224                 for (Classification classif : classificationList){
 225                     if (classif.getTitleCache().equalsIgnoreCase(name) && classif.getCitation().equals(ref)) {
 226                         classification=classif;
 227 //                        System.out.println("FIND SAME CLASSIF");
 228                     }
 229                 }
 230                 if (classification == null){
 231                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 232                 }
 233                 //                if (state.getConfig().getClassificationUuid() != null) {
 234                 //                    classification.setUuid(state.getConfig().getClassificationUuid());
 235                 //                }
 236                 save(classification, state);
 237             }
 238         }
 239
 240         InputStream source = state.getConfig().getSource();
 241         NodeList unitsList = getUnitsNodeList(source);
 242
 243         if (unitsList != null) {
 244             String message = "nb units to insert: " + unitsList.getLength();
 245             logger.info(message);
 246             updateProgress(state, message);
 247
 248             dataHolder = new Abcd206DataHolder();
 249
 250             Abcd206XMLFieldGetter abcdFieldGetter = new Abcd206XMLFieldGetter(dataHolder, prefix);
 251
 252             prepareCollectors(state, unitsList, abcdFieldGetter);
 253
 254             associationRefs = new ArrayList<OriginalSourceBase<?>>();
 255             descriptionRefs = new ArrayList<OriginalSourceBase<?>>();
 256             derivedUnitSources = new ArrayList<OriginalSourceBase<?>>();
 257
 258             for (int i = 0; i < unitsList.getLength(); i++) {
 259                 System.out.println("------------------------------------------------------------------------------------------");
 260
 261                 this.setUnitPropertiesXML( (Element) unitsList.item(i), abcdFieldGetter);
 262                 //                              refreshTransaction(state);
 263                 this.handleSingleUnit(state);
 264
 265                 // compare the ABCD elements added in to the CDM and the
 266                 // unhandled ABCD elements
 267                 //compareABCDtoCDM(sourceName, dataHolder.knownABCDelements, abcdFieldGetter);
 268
 269                 // reset the ABCD elements added in CDM
 270                 // knownABCDelements = new ArrayList<String>();
 271                 dataHolder.allABCDelements = new HashMap<String, String>();
 272             }
 273             getReferenceService().deduplicate(Reference.class, null, null);
 274             getClassificationService().deduplicate(Classification.class, null, null);
 275         }
 276         commitTransaction(state.getTx());
 277         return;
 278
 279     }
 280
 281
 282     protected NodeList getUnitsNodeList(URI source) {
 283         try {
 284             InputStream is = UriUtils.getInputStream(source);
 285             return getUnitsNodeList(is);
 286         } catch (Exception e) {
 287             logger.warn(e);
 288             throw new RuntimeException(e);
 289         }
 290     }
 291
 292     /**
 293      * Return the list of root nodes for an ABCD 2.06 XML file
 294      * @param fileName: the file's location
 295      * @return the list of root nodes ("Unit")
 296      */
 297     protected NodeList getUnitsNodeList(InputStream inputStream) {
 298         NodeList unitList = null;
 299         try {
 300             DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 301             DocumentBuilder builder = factory.newDocumentBuilder();
 302
 303             Document document = builder.parse(inputStream);
 304             Element root = document.getDocumentElement();
 305             unitList = root.getElementsByTagName("Unit");
 306             if (unitList.getLength() == 0) {
 307                 unitList = root.getElementsByTagName("abcd:Unit");
 308                 prefix = "abcd:";
 309             }
 310         } catch (Exception e) {
 311             logger.warn(e);
 312         }
 313         return unitList;
 314     }
 315
 316     /**
 317      * Handle a single unit
 318      * @param state
 319      */
 320     @SuppressWarnings("rawtypes")
 321     private void handleSingleUnit(Abcd206ImportState state) {
 322         if (DEBUG) {
 323             logger.info("handleSingleUnit "+ref);
 324         }
 325         try {
 326             updateProgress(state, "Importing data for unit: " + dataHolder.unitID);
 327
 328             // create facade
 329             DerivedUnitFacade derivedUnitFacade = getFacade();
 330             derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 331
 332             /**
 333              * GATHERING EVENT
 334              */
 335             // gathering event
 336             ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
 337             if(cdmAppController==null){
 338                 cdmAppController = this;
 339             }
 340             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(cdmAppController.getTermService(), dataHolder.locality, dataHolder.languageIso,
 341                     dataHolder.longitude, dataHolder.latitude, dataHolder.gatheringAgentList, dataHolder.gatheringTeamList,state.getConfig());
 342
 343             // country
 344             UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea();
 345             //  unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(), getTermService());
 346             unitsGatheringArea.setParams(dataHolder.isocountry, dataHolder.country, state.getConfig(), cdmAppController.getTermService(), getOccurrenceService());
 347
 348             DefinedTermBase<?> areaCountry =  unitsGatheringArea.getCountry();
 349
 350             // other areas
 351             unitsGatheringArea = new UnitsGatheringArea();
 352             //            unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(),getTermService());
 353             unitsGatheringArea.setAreas(dataHolder.namedAreaList,state.getConfig(), cdmAppController.getTermService());
 354             ArrayList<DefinedTermBase> nas = unitsGatheringArea.getAreas();
 355             for (DefinedTermBase namedArea : nas) {
 356                 unitsGatheringEvent.addArea(namedArea);
 357             }
 358
 359             // copy gathering event to facade
 360             GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 361             derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 362             derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 363             derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 364             derivedUnitFacade.setCountry((NamedArea)areaCountry);
 365
 366             for(DefinedTermBase<?> area:unitsGatheringArea.getAreas()){
 367                 derivedUnitFacade.addCollectingArea((NamedArea) area);
 368             }
 369             //            derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 370             // TODO exsiccatum
 371
 372             // add fieldNumber
 373             derivedUnitFacade.setFieldNumber(NB(dataHolder.fieldNumber));
 374
 375             // //add Multimedia URLs
 376             if (dataHolder.multimediaObjects.size() != -1) {
 377                 for (String multimediaObject : dataHolder.multimediaObjects) {
 378                     Media media;
 379                     try {
 380                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA);
 381                         derivedUnitFacade.addDerivedUnitMedia(media);
 382                     } catch (MalformedURLException e) {
 383                         // TODO Auto-generated catch block
 384                         e.printStackTrace();
 385                     }
 386
 387                 }
 388             }
 389
 390             //                  /*
 391             //                   * merge AND STORE DATA
 392             //                   */
 393             //                  getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 394             //
 395             //                  for (NamedArea area : otherAreas) {
 396             //                          getTermService().saveOrUpdate(area);// merge it sooner (foreach area)
 397             //                  }
 398
 399             save(unitsGatheringEvent.getLocality(), state);
 400
 401             // handle collection data
 402             setCollectionData(state, derivedUnitFacade);
 403
 404             //Reference stuff
 405             SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 406             Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
 407
 408             dataHolder.docSources = new ArrayList<String>();
 409             for (String[] fullReference : dataHolder.referenceList) {
 410                 String strReference=fullReference[0];
 411                 String citationDetail = fullReference[1];
 412                 String citationURL = fullReference[2];
 413
 414                 if (!citationURL.isEmpty()) {
 415                     citationDetail+=", "+citationURL;
 416                 }
 417
 418                 Reference<?> reference = ReferenceFactory.newGeneric();
 419                 reference.setTitle(strReference);
 420
 421                 IdentifiableSource sour = getIdentifiableSource(reference,citationDetail);
 422
 423                 try{
 424                     if (sour.getCitation() != null){
 425                         if(StringUtils.isNotBlank(sour.getCitationMicroReference())) {
 426                             dataHolder.docSources.add(sour.getCitation().getTitleCache()+ "---"+sour.getCitationMicroReference());
 427                         } else {
 428                             dataHolder.docSources.add(sour.getCitation().getTitleCache());
 429                         }
 430                     }
 431                 }catch(Exception e){
 432                     logger.warn("oups");
 433                 }
 434                 reference.addSource(sour);
 435                 save(reference, state);
 436             }
 437
 438
 439             List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 440             List<DescriptionElementSource> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 441
 442             Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
 443             if(issTmp2!=null) {
 444                 osbSet.addAll(issTmp2);
 445             }
 446             if(issTmp!=null) {
 447                 osbSet.addAll(issTmp);
 448             }
 449
 450
 451             for( OriginalSourceBase<?> osb:osbSet) {
 452                 if(osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
 453                     try{
 454                         sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
 455                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
 456                 } else{
 457                     try{
 458                         sourceMap.put(osb.getCitation().getTitleCache(),osb);
 459                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
 460                 }
 461             }
 462
 463             if( state.getConfig().isInteractWithUser()){
 464                 List<OriginalSourceBase<?>>sources=null;
 465                 if(!derivedUnitSourcesSet){
 466                     sources= sui.askForSource(sourceMap, "the unit itself","",getReferenceService(), dataHolder.docSources);
 467                     derivedUnitSources=sources;
 468                     derivedUnitSourcesSet=true;
 469                 }
 470                 else{
 471                     sources=derivedUnitSources;
 472                 }
 473 //                System.out.println("nb sources: "+sources.size());
 474 //                System.out.println("derivedunitfacade : "+derivedUnitFacade.getTitleCache());
 475                 for (OriginalSourceBase<?> sour:sources){
 476                     if(sour.isInstanceOf(IdentifiableSource.class)){
 477                         if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
 478 //                            System.out.println("add source to derivedunitfacade1 "+derivedUnitFacade.getTitleCache());
 479                             derivedUnitFacade.addSource((IdentifiableSource)sour.clone());
 480                         }
 481                     }else{
 482                         if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
 483 //                            System.out.println("add source to derivedunitfacade2 "+derivedUnitFacade.getTitleCache());
 484                             derivedUnitFacade.addSource(OriginalSourceType.Import,sour.getCitation(),sour.getCitationMicroReference(), ioName);
 485                         }
 486                     }
 487                 }
 488             }else{
 489                 for (OriginalSourceBase<?> sr : sourceMap.values()){
 490                     if(sr.isInstanceOf(IdentifiableSource.class)){
 491                         if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
 492 //                            System.out.println("add source to derivedunitfacade3 "+derivedUnitFacade.getTitleCache());
 493                             derivedUnitFacade.addSource((IdentifiableSource)sr.clone());
 494                         }
 495                     }else{
 496                         if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
 497 //                            System.out.println("add source to derivedunitfacade4 "+derivedUnitFacade.getTitleCache());
 498                             derivedUnitFacade.addSource(OriginalSourceType.Import,sr.getCitation(),sr.getCitationMicroReference(), ioName);
 499                         }
 500                     }
 501                 }
 502             }
 503
 504             save(derivedUnitBase, state);
 505
 506             // handle identifications
 507             handleIdentifications(state, derivedUnitFacade);
 508
 509             if(DEBUG) {
 510                 logger.info("saved ABCD specimen ...");
 511             }
 512
 513         } catch (Exception e) {
 514             logger.warn("Error when reading record!!");
 515             e.printStackTrace();
 516             state.setUnsuccessfull();
 517         }
 518
 519         return;
 520     }
 521
 522     /**
 523      * @param derivedUnitFacade
 524      * @param sour
 525      * @return
 526      */
 527     private boolean sourceNotLinkedToElement(DerivedUnitFacade derivedUnitFacade, OriginalSourceBase<?> source) {
 528         Set<IdentifiableSource> linkedSources = derivedUnitFacade.getSources();
 529         for (IdentifiableSource is:linkedSources){
 530             if (is.getCitation()!=null && source.getCitation()!=null &&
 531                     is.getCitation().getTitleCache().equalsIgnoreCase(source.getCitation().getTitleCache())){
 532                 String isDetail =  is.getCitationMicroReference();
 533                 if ((StringUtils.isBlank(isDetail) && StringUtils.isBlank(source.getCitationMicroReference()))
 534                         || (isDetail != null && isDetail.equalsIgnoreCase(source.getCitationMicroReference())) ) {
 535                     return false;
 536                 }
 537             }
 538         }
 539         return true;
 540     }
 541
 542     /**
 543      * @param reference
 544      * @param citationDetail
 545      * @return
 546      */
 547     //FIXME this method is highly critical, because
 548     //  * it will have serious performance and memory problems with large databases
 549     //        (databases may easily have >1 Mio source records)
 550     //  * it does not make sense to search for existing sources and then clone them
 551     //    we need to search for existing references instead and use them (if exist)
 552     //    for our new source.
 553     private IdentifiableSource getIdentifiableSource(Reference<?> reference, String citationDetail) {
 554
 555         List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 556
 557
 558         if (reference != null){
 559             try {
 560                 for (OriginalSourceBase<?> osb: issTmp){
 561                     if (osb.getCitation() != null && reference!=null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 562                         String osbDetail = osb.getCitationMicroReference();
 563                         if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 564                                 || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 565 //                            System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 566                             return (IdentifiableSource) osb.clone();
 567                         }
 568                     }
 569                 }
 570             } catch (CloneNotSupportedException e) {
 571                 throw new RuntimeException(e);
 572             } catch (Exception e1){
 573                 e1.printStackTrace();
 574             }
 575         }
 576
 577         IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 578         return sour;
 579     }
 580
 581     //    /**
 582     //     * @param reference
 583     //     * @param citationDetail
 584     //     * @return
 585     //     */
 586     //    private DescriptionElementSource getDescriptionSource(Reference<?> reference, String citationDetail) {
 587     //
 588     //        List<OriginalSourceBase> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 589     //
 590     //        try {
 591     //            for (OriginalSourceBase<?> osb:issTmp2){
 592     //                if (osb.getCitation().equals(reference) && osb.getCitationMicroReference().equalsIgnoreCase(citationDetail)) {
 593     //                    return (DescriptionElementSource) osb.clone();
 594     //                }
 595     //            }
 596     //        } catch (CloneNotSupportedException e) {
 597     //            // TODO Auto-generated catch block
 598     //            e.printStackTrace();
 599     //        }
 600     //
 601     //        DescriptionElementSource sour = DescriptionElementSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 602     //        return sour;
 603     //    }
 604
 605
 606     /**
 607      * Very fast and dirty implementation to allow handling of transient objects as described in
 608      * https://dev.e-taxonomy.eu/trac/ticket/3726
 609      *
 610      * Not yet complete.
 611      *
 612      * @param cdmBase
 613      * @param state
 614      */
 615     private void save(CdmBase cdmBase, Abcd206ImportState state) {
 616         ICdmApplicationConfiguration cdmRepository = state.getConfig().getCdmAppController();
 617         if (cdmRepository == null){
 618             cdmRepository = this;
 619         }
 620
 621         if (cdmBase.isInstanceOf(LanguageString.class)){
 622             cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 623         }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 624             cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 625         }else if (cdmBase.isInstanceOf(Reference.class)){
 626             cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 627         }else if (cdmBase.isInstanceOf(Classification.class)){
 628             cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 629         }else if (cdmBase.isInstanceOf(AgentBase.class)){
 630             cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 631         }else if (cdmBase.isInstanceOf(Collection.class)){
 632             cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 633         }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 634             cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 635         }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 636             cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 637         }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 638             cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 639         }else{
 640             throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 641         }
 642
 643     }
 644
 645     /**
 646      * setCollectionData : store the collection object into the
 647      * derivedUnitFacade
 648      *
 649      * @param state
 650      */
 651     private void setCollectionData(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
 652         // set catalogue number (unitID)
 653         derivedUnitFacade.setCatalogNumber(NB(dataHolder.unitID));
 654         derivedUnitFacade.setAccessionNumber(NB(dataHolder.accessionNumber));
 655         // derivedUnitFacade.setCollectorsNumber(NB(dataHolder.collectorsNumber));
 656
 657         /*
 658          * INSTITUTION & COLLECTION
 659          */
 660         // manage institution
 661         Institution institution = this.getInstitution(NB(dataHolder.institutionCode), state);
 662         // manage collection
 663         Collection collection = this.getCollection(institution, NB(dataHolder.collectionCode), state);
 664         // link specimen & collection
 665         derivedUnitFacade.setCollection(collection);
 666     }
 667
 668     /**
 669      * getFacade : get the DerivedUnitFacade based on the recordBasis
 670      *
 671      * @return DerivedUnitFacade
 672      */
 673     private DerivedUnitFacade getFacade() {
 674         if(DEBUG) {
 675             logger.info("getFacade()");
 676         }
 677         SpecimenOrObservationType type = null;
 678
 679         // create specimen
 680         if (NB((dataHolder.recordBasis)) != null) {
 681             if (dataHolder.recordBasis.toLowerCase().startsWith("s") || dataHolder.recordBasis.toLowerCase().indexOf("specimen")>-1) {// specimen
 682                 type = SpecimenOrObservationType.PreservedSpecimen;
 683             }
 684             if (dataHolder.recordBasis.toLowerCase().startsWith("o") ||dataHolder.recordBasis.toLowerCase().indexOf("observation")>-1 ) {
 685                 type = SpecimenOrObservationType.Observation;
 686             }
 687             if (dataHolder.recordBasis.toLowerCase().indexOf("fossil")>-1){
 688                 type = SpecimenOrObservationType.Fossil;
 689             }
 690             if (dataHolder.recordBasis.toLowerCase().indexOf("living")>-1) {
 691                 type = SpecimenOrObservationType.LivingSpecimen;
 692             }
 693             if (type == null) {
 694                 logger.info("The basis of record does not seem to be known: " + dataHolder.recordBasis);
 695                 type = SpecimenOrObservationType.DerivedUnit;
 696             }
 697             // TODO fossils?
 698         } else {
 699             logger.info("The basis of record is null");
 700             type = SpecimenOrObservationType.DerivedUnit;
 701         }
 702         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 703         return derivedUnitFacade;
 704     }
 705
 706     private void getCollectorsFromXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 707         NodeList group;
 708
 709         group = root.getChildNodes();
 710         for (int i = 0; i < group.getLength(); i++) {
 711             if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 712                 group = group.item(i).getChildNodes();
 713                 break;
 714             }
 715         }
 716         dataHolder.gatheringAgentList = new ArrayList<String>();
 717         dataHolder.gatheringTeamList = new ArrayList<String>();
 718         abcdFieldGetter.getType(root);
 719         abcdFieldGetter.getGatheringPeople(root);
 720     }
 721
 722     /**
 723      * Store the unit's properties into variables Look which unit is the
 724      * preferred one Look what kind of name it is supposed to be, for the
 725      * parsing (Botanical, Zoological)
 726      *
 727      * @param racine: the root node for a single unit
 728      */
 729     private void setUnitPropertiesXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 730         try {
 731             NodeList group;
 732
 733             group = root.getChildNodes();
 734             for (int i = 0; i < group.getLength(); i++) {
 735                 if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 736                     group = group.item(i).getChildNodes();
 737                     break;
 738                 }
 739             }
 740             dataHolder.identificationList = new ArrayList<Identification>();
 741             dataHolder.statusList = new ArrayList<SpecimenTypeDesignationStatus>();
 742             dataHolder.atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
 743             dataHolder.referenceList = new ArrayList<String[]>();
 744             dataHolder.multimediaObjects = new ArrayList<String>();
 745
 746             abcdFieldGetter.getScientificNames(group);
 747             abcdFieldGetter.getType(root);
 748
 749             if(DEBUG) {
 750                 logger.info("this.identificationList "+dataHolder.identificationList.toString());
 751             }
 752             abcdFieldGetter.getIDs(root);
 753             abcdFieldGetter.getRecordBasis(root);
 754             abcdFieldGetter.getMultimedia(root);
 755             abcdFieldGetter.getNumbers(root);
 756             abcdFieldGetter.getGeolocation(root);
 757             abcdFieldGetter.getGatheringPeople(root);
 758             boolean referencefound = abcdFieldGetter.getReferences(root);
 759             if (!referencefound) {
 760                 String[]a = {ref.getTitleCache(),"",""};
 761                 dataHolder.referenceList.add(a);
 762             }
 763
 764         } catch (Exception e) {
 765             logger.info("Error occured while parsing XML file" + e);
 766         }
 767     }
 768
 769     /**
 770      * Look if the Institution does already exist
 771      * @param institutionCode: a string with the institutioncode
 772      * @param config : the configurator
 773      * @return the Institution (existing or new)
 774      */
 775     @SuppressWarnings("rawtypes")
 776     private Institution getInstitution(String institutionCode, Abcd206ImportState state) {
 777         Institution institution=null;
 778         List<Institution> institutions;
 779         try {
 780             institutions = getAgentService().list(Institution.class, null, null, null, null);
 781         } catch (Exception e) {
 782             institutions = new ArrayList<Institution>();
 783             logger.warn(e);
 784         }
 785         if (institutions.size() > 0 && state.getConfig().isReUseExistingMetadata()) {
 786             for (Institution institut:institutions){
 787                 try{
 788                     if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 789                         institution=institut;
 790                     }
 791                 }catch(Exception e){logger.warn("no institution code in the db");}
 792             }
 793         }
 794         if(DEBUG) {
 795             if(institution !=null) {
 796                 logger.info("getinstitution " + institution.toString());
 797             }
 798         }
 799         if (institution == null){
 800             // create institution
 801             institution = Institution.NewInstance();
 802             institution.setCode(institutionCode);
 803             institution.setTitleCache(institutionCode, true);
 804         }
 805         save(institution, state);
 806         return institution;
 807     }
 808
 809     /**
 810      * Look if the Collection does already exist
 811      * @param collectionCode
 812      * @param collectionCode: a string
 813      * @param config : the configurator
 814      * @return the Collection (existing or new)
 815      */
 816     private Collection getCollection(Institution institution, String collectionCode, Abcd206ImportState state) {
 817         Collection collection = null;
 818         List<Collection> collections;
 819         try {
 820             collections = getCollectionService().list(Collection.class, null, null, null, null);
 821         } catch (Exception e) {
 822             collections = new ArrayList<Collection>();
 823         }
 824         if (collections.size() > 0 && state.getConfig().isReUseExistingMetadata()) {
 825             for (Collection coll:collections){
 826                 if (coll.getInstitute() != null) {
 827                     if (coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 828                         collection=coll;
 829                     }
 830                 }
 831             }
 832         }
 833
 834         if(collection == null){
 835             collection =Collection.NewInstance();
 836             collection.setCode(collectionCode);
 837             collection.setInstitute(institution);
 838             collection.setTitleCache(collectionCode);
 839         }
 840         save(collection, state);
 841         return collection;
 842     }
 843
 844
 845     /**
 846      * join DeterminationEvent to the Taxon Object
 847      * @param state : the ABCD import state
 848      * @param taxon: the current Taxon
 849      * @param preferredFlag :if the current name is preferred
 850      * @param derivedFacade : the derived Unit Facade
 851      */
 852     @SuppressWarnings("rawtypes")
 853     private void linkDeterminationEvent(Abcd206ImportState state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
 854         Abcd206ImportConfigurator config = state.getConfig();
 855         if(DEBUG){
 856             logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
 857         }
 858
 859         DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 860         determinationEvent.setTaxon(taxon);
 861         determinationEvent.setPreferredFlag(preferredFlag);
 862
 863         determinationEvent.setIdentifiedUnit(derivedUnitBase);
 864         derivedUnitBase.addDetermination(determinationEvent);
 865
 866         try {
 867             if(DEBUG){
 868                 logger.info("NB TYPES INFO: "+ dataHolder.statusList.size());
 869             }
 870             for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : dataHolder.statusList) {
 871                 if (specimenTypeDesignationstatus != null) {
 872                     if(DEBUG){
 873                         logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
 874                     }
 875
 876                     ICdmApplicationConfiguration cdmAppController = config.getCdmAppController();
 877                     if(cdmAppController == null){
 878                         cdmAppController = this;
 879                     }
 880                     specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
 881                     //Designation
 882                     TaxonNameBase<?,?> name = taxon.getName();
 883                     SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
 884
 885                     designation.setTypeStatus(specimenTypeDesignationstatus);
 886                     designation.setTypeSpecimen(derivedUnitBase);
 887                     name.addTypeDesignation(designation, true);
 888                 }
 889             }
 890         } catch (Exception e) {
 891             logger.warn("PB addding SpecimenType " + e);
 892         }
 893
 894         for (String[] fullReference : dataHolder.referenceList) {
 895             try{
 896 //                System.out.println(fullReference);
 897                 List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
 898
 899                 String strReference=fullReference[0];
 900                 String citationDetail = fullReference[1];
 901                 String citationURL = fullReference[2];
 902
 903                 if (isNotBlank(strReference)){
 904                     Reference<?> reference = null;
 905                     for (Reference<?> refe: references) {
 906                         if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
 907                             reference =refe;
 908                             break;
 909                         }
 910                     }
 911                     if (reference ==null){
 912                         reference = ReferenceFactory.newGeneric();
 913                         /*<<<<<<< .courant
 914                     reference.setTitleCache(strReference);
 915                     System.out.println("reference hasproblem2 "+reference.hasProblem());
 916                     IdentifiableSource sour = IdentifiableSource.NewInstance(reference,citationDetail);
 917                     getReferenceService().saveOrUpdate(sour.getCitation());
 918 =======*/
 919                         reference.setTitleCache(strReference, true);
 920                         save(reference, state);
 921                     }
 922                     determinationEvent.addReference(reference);
 923                 }
 924             }catch(Exception e){logger.warn("pv getReferenceList "+e);}
 925         }
 926         save(derivedUnitBase, state);
 927
 928         if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations()) {
 929             if(DEBUG){
 930                 logger.info("isDoCreateIndividualsAssociations");
 931             }
 932
 933             makeIndividualsAssociation(state, taxon, determinationEvent);
 934             save(derivedUnitBase, state);
 935         }
 936     }
 937
 938     /**
 939      * create and link each association (specimen, observation..) to the accepted taxon
 940      * @param state : the ABCD import state
 941      * @param taxon: the current Taxon
 942      * @param determinationEvent:the determinationevent
 943      */
 944     @SuppressWarnings("unused")
 945     private void makeIndividualsAssociation(Abcd206ImportState state, Taxon taxon, DeterminationEvent determinationEvent) {
 946         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 947
 948         if (DEBUG) {
 949             System.out.println("MAKE INDIVIDUALS ASSOCIATION");
 950         }
 951
 952         TaxonDescription taxonDescription = null;
 953         Set<TaxonDescription> descriptions= taxon.getDescriptions();
 954         if (state.getConfig().isInteractWithUser()){
 955             if(!descriptionGroupSet){
 956                 taxonDescription = sui.askForDescriptionGroup(descriptions);
 957                 descriptionGroup=taxonDescription;
 958                 descriptionGroupSet=true;
 959             }else{
 960                 taxonDescription=descriptionGroup;
 961             }
 962         } else {
 963             for (TaxonDescription description : descriptions){
 964                 Set<IdentifiableSource> sources =  description.getTaxon().getSources();
 965                 sources.addAll(description.getSources());
 966                 for (IdentifiableSource source:sources){
 967                     if(ref.equals(source.getCitation())) {
 968                         taxonDescription = description;
 969                     }
 970                 }
 971             }
 972         }
 973         if (taxonDescription == null){
 974             taxonDescription = TaxonDescription.NewInstance(taxon, false);
 975             if(sourceNotLinkedToElement(taxonDescription,ref,null)) {
 976                 taxonDescription.addSource(OriginalSourceType.Import, null, null, ref, null);
 977             }
 978             descriptionGroup=taxonDescription;
 979             taxon.addDescription(taxonDescription);
 980         }
 981
 982         //PREPARE REFERENCE QUESTIONS
 983
 984         Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
 985
 986         List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 987         List<DescriptionElementSource> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 988
 989         Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
 990         if(issTmp2!=null) {
 991             osbSet.addAll(issTmp2);
 992         }
 993         if(issTmp!=null) {
 994             osbSet.addAll(issTmp);
 995         }
 996
 997
 998         for( OriginalSourceBase<?> osb:osbSet) {
 999             if(osb.getCitationMicroReference() !=null && !osb.getCitationMicroReference().isEmpty()) {
1000                 try{
1001                     sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1002                 }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1003             } else{
1004                 try{
1005                     sourceMap.put(osb.getCitation().getTitleCache(),osb);
1006                 }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1007             }
1008         }
1009
1010         if (state.getConfig().isInteractWithUser()){
1011             List<OriginalSourceBase<?>> res = null;
1012             if(!descriptionSourcesSet){
1013                 res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1014                         "The current reference is "+ref.getTitleCache(),getReferenceService(), dataHolder.docSources);
1015                 descriptionRefs=res;
1016                 descriptionSourcesSet=true;
1017             }
1018             else{
1019                 res=descriptionRefs;
1020             }
1021             if(res !=null) {
1022                 for (OriginalSourceBase<?> sour:res){
1023                     if(sour.isInstanceOf(IdentifiableSource.class)){
1024                         try {
1025                             if(sourceNotLinkedToElement(taxonDescription,sour)) {
1026                                 taxonDescription.addSource((IdentifiableSource)sour.clone());
1027                             }
1028                         } catch (CloneNotSupportedException e) {
1029                             logger.warn("no cloning?");
1030                         }
1031                     }else{
1032                         if(sourceNotLinkedToElement(taxonDescription,sour)) {
1033                             taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1034                         }
1035                     }
1036                 }
1037             }
1038         }
1039         else {
1040             if(sourceNotLinkedToElement(taxonDescription,ref,null)) {
1041                 taxonDescription.addSource(OriginalSourceType.Import,null, null, ref, null);
1042             }
1043         }
1044         descriptionGroup=taxonDescription;
1045
1046         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1047         Feature feature = makeFeature(derivedUnitBase);
1048         indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
1049         indAssociation.setFeature(feature);
1050
1051         if (state.getConfig().isInteractWithUser()){
1052             sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1053
1054             issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1055             issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1056
1057             osbSet = new HashSet<OriginalSourceBase>();
1058             if(issTmp2!=null) {
1059                 osbSet.addAll(issTmp2);
1060             }
1061             if(issTmp!=null) {
1062                 osbSet.addAll(issTmp);
1063             }
1064
1065
1066             for( OriginalSourceBase<?> osb:osbSet) {
1067                 if(osb.getCitationMicroReference() !=null && !osb.getCitationMicroReference().isEmpty()) {
1068                     try{
1069                         sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1070                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1071                 } else{
1072                     try{
1073                         sourceMap.put(osb.getCitation().getTitleCache(),osb);
1074                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1075                 }
1076             }
1077
1078             List<OriginalSourceBase<?>> sources =null;
1079             if(!associationSourcesSet) {
1080                 sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1081                         getReferenceService(),dataHolder.docSources);
1082                 associationRefs=sources;
1083                 associationSourcesSet=true;
1084             }
1085             else{
1086                 sources=associationRefs;
1087             }
1088             if(sources !=null) {
1089                 for (OriginalSourceBase<?> source: sources) {
1090                     if(source !=null) {
1091                         if(source.isInstanceOf(DescriptionElementSource.class)){
1092                             try {
1093                                 if(sourceNotLinkedToElement(indAssociation,source)) {
1094                                     indAssociation.addSource((DescriptionElementSource)source.clone());
1095                                 }
1096                             } catch (CloneNotSupportedException e) {
1097                                 logger.warn("clone forbidden?");
1098                             }
1099                         }else{
1100                             if(sourceNotLinkedToElement(indAssociation,source)) {
1101                                 indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1102                             }
1103                             try {
1104                                 if(sourceNotLinkedToElement(derivedUnitBase, source)) {
1105                                     derivedUnitBase.addSource((IdentifiableSource) source.clone());
1106                                 }
1107                             } catch (CloneNotSupportedException e) {
1108                                 // TODO Auto-generated catch block
1109                                 e.printStackTrace();
1110                             }
1111                         }
1112
1113                     }
1114                 }
1115             }
1116         }else {
1117             if(sourceNotLinkedToElement(indAssociation,ref,null)) {
1118                 indAssociation.addSource(OriginalSourceType.Import,null, null, ref, null);
1119             }
1120             if(sourceNotLinkedToElement(derivedUnitBase, ref,null)) {
1121                 derivedUnitBase.addSource(OriginalSourceType.Import,null, null, ref, null);
1122             }
1123             for (Reference<?> citation : determinationEvent.getReferences()) {
1124                 if(sourceNotLinkedToElement(indAssociation,citation,null))
1125                 {
1126                     indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1127                 }
1128                 if(sourceNotLinkedToElement(derivedUnitBase, ref,null)) {
1129                     derivedUnitBase.addSource(OriginalSourceType.Import,null, null, ref, null);
1130                 }
1131             }
1132         }
1133
1134         taxonDescription.addElement(indAssociation);
1135
1136         save(taxonDescription, state);
1137         save(taxon, state);
1138     }
1139
1140
1141
1142     /**
1143      * @param derivedUnitBase2
1144      * @param ref2
1145      * @param object
1146      * @return
1147      */
1148     private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference<?> b, String d) {
1149         Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1150         for (IdentifiableSource is:linkedSources){
1151             Reference a = is.getCitation();
1152             String c = is.getCitationMicroReference();
1153
1154             boolean refMatch=false;
1155             boolean microMatch=false;
1156
1157             try{
1158                 if (a==null && b==null) {
1159                     refMatch=true;
1160                 }
1161                 if (a!=null && b!=null) {
1162                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1163                         refMatch=true;
1164                     }
1165                 }
1166             }catch(Exception e){}
1167
1168
1169             try{
1170                 if (c==null && d==null) {
1171                     microMatch=true;
1172                 }
1173                 if(c!=null && d!=null) {
1174                     if(c.equalsIgnoreCase(d)) {
1175                         microMatch=true;
1176                     }
1177                 }
1178             }
1179             catch(Exception e){}
1180
1181             if (microMatch && refMatch) {
1182                 return false;
1183             }
1184
1185
1186         }
1187         return true;
1188     }
1189
1190     /**
1191      * @param derivedUnitBase2
1192      * @param source
1193      * @return
1194      */
1195     private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, OriginalSourceBase<?> source) {
1196         Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1197         for (IdentifiableSource is:linkedSources){
1198             Reference a = is.getCitation();
1199             Reference b = source.getCitation();
1200             String c = is.getCitationMicroReference();
1201             String d = source.getCitationMicroReference();
1202
1203             boolean refMatch=false;
1204             boolean microMatch=false;
1205
1206             try{
1207                 if (a==null && b==null) {
1208                     refMatch=true;
1209                 }
1210                 if (a!=null && b!=null) {
1211                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1212                         refMatch=true;
1213                     }
1214                 }
1215             }catch(Exception e){}
1216
1217
1218             try{
1219                 if (c==null && d==null) {
1220                     microMatch=true;
1221                 }
1222                 if(c!=null && d!=null) {
1223                     if(c.equalsIgnoreCase(d)) {
1224                         microMatch=true;
1225                     }
1226                 }
1227             }
1228             catch(Exception e){}
1229
1230             if (microMatch && refMatch) {
1231                 return false;
1232             }
1233
1234
1235         }
1236         return true;
1237     }
1238
1239     /**
1240      * @param indAssociation
1241      * @param ref2
1242      * @param object
1243      * @return
1244      */
1245     private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, Reference<?> a, String d) {
1246         Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1247         for (DescriptionElementSource is:linkedSources){
1248             Reference b = is.getCitation();
1249             String c = is.getCitationMicroReference();
1250
1251             boolean refMatch=false;
1252             boolean microMatch=false;
1253
1254             try{
1255                 if (a==null && b==null) {
1256                     refMatch=true;
1257                 }
1258                 if (a!=null && b!=null) {
1259                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1260                         refMatch=true;
1261                     }
1262                 }
1263             }catch(Exception e){}
1264
1265
1266             try{
1267                 if (c==null && d==null) {
1268                     microMatch=true;
1269                 }
1270                 if(c!=null && d!=null) {
1271                     if(c.equalsIgnoreCase(d)) {
1272                         microMatch=true;
1273                     }
1274                 }
1275             }
1276             catch(Exception e){}
1277
1278             if (microMatch && refMatch) {
1279                 return false;
1280             }
1281         }
1282         return true;
1283     }
1284
1285     /**
1286      * @param taxonDescription
1287      * @param ref2
1288      * @param object
1289      * @return
1290      */
1291     private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, Reference<?> a, String d) {
1292         Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1293         for (IdentifiableSource is:linkedSources){
1294             Reference b = is.getCitation();
1295             String c = is.getCitationMicroReference();
1296
1297             boolean refMatch=false;
1298             boolean microMatch=false;
1299
1300             try{
1301                 if (a==null && b==null) {
1302                     refMatch=true;
1303                 }
1304                 if (a!=null && b!=null) {
1305                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1306                         refMatch=true;
1307                     }
1308                 }
1309             }catch(Exception e){}
1310
1311
1312             try{
1313                 if (c==null && d==null) {
1314                     microMatch=true;
1315                 }
1316                 if(c!=null && d!=null) {
1317                     if(c.equalsIgnoreCase(d)) {
1318                         microMatch=true;
1319                     }
1320                 }
1321             }
1322             catch(Exception e){}
1323
1324             if (microMatch && refMatch) {
1325                 return false;
1326             }
1327         }
1328         return true;
1329     }
1330
1331     /**
1332      * @param indAssociation
1333      * @param source
1334      * @return
1335      */
1336     private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, OriginalSourceBase<?> source) {
1337         Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1338         for (DescriptionElementSource is:linkedSources){
1339             Reference a = is.getCitation();
1340             Reference b = source.getCitation();
1341             String c = is.getCitationMicroReference();
1342             String d = source.getCitationMicroReference();
1343
1344             boolean refMatch=false;
1345             boolean microMatch=false;
1346
1347             try{
1348                 if (a==null && b==null) {
1349                     refMatch=true;
1350                 }
1351                 if (a!=null && b!=null) {
1352                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1353                         refMatch=true;
1354                     }
1355                 }
1356             }catch(Exception e){}
1357
1358
1359             try{
1360                 if (c==null && d==null) {
1361                     microMatch=true;
1362                 }
1363                 if(c!=null && d!=null) {
1364                     if(c.equalsIgnoreCase(d)) {
1365                         microMatch=true;
1366                     }
1367                 }
1368             }
1369             catch(Exception e){}
1370
1371             if (microMatch && refMatch) {
1372                 return false;
1373             }
1374         }
1375         return true;
1376     }
1377
1378     /**
1379      * @param taxonDescription
1380      * @param sour
1381      * @return
1382      */
1383     private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, OriginalSourceBase<?> sour) {
1384         Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1385         for (IdentifiableSource is:linkedSources){
1386             Reference a = is.getCitation();
1387             Reference b = sour.getCitation();
1388             String c = is.getCitationMicroReference();
1389             String d = sour.getCitationMicroReference();
1390
1391             boolean refMatch=false;
1392             boolean microMatch=false;
1393
1394             try{
1395                 if (a==null && b==null) {
1396                     refMatch=true;
1397                 }
1398                 if (a!=null && b!=null) {
1399                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1400                         refMatch=true;
1401                     }
1402                 }
1403             }catch(Exception e){}
1404
1405
1406             try{
1407                 if (c==null && d==null) {
1408                     microMatch=true;
1409                 }
1410                 if(c!=null && d!=null) {
1411                     if(c.equalsIgnoreCase(d)) {
1412                         microMatch=true;
1413                     }
1414                 }
1415             }
1416             catch(Exception e){}
1417
1418             if (microMatch && refMatch) {
1419                 return false;
1420             }
1421
1422
1423         }
1424         return true;
1425     }
1426
1427     /**
1428      * look for the Feature object (FieldObs, Specimen,...)
1429      * @param unit : a specimen or obersvation base
1430      * @return the corresponding Feature
1431      */
1432     private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1433         SpecimenOrObservationType type = unit.getRecordBasis();
1434
1435
1436
1437         if (type.isFeatureObservation()){
1438             return Feature.OBSERVATION();
1439         }else if (type.isFeatureSpecimen()){
1440             return Feature.SPECIMEN();
1441         }else if (type == SpecimenOrObservationType.DerivedUnit){
1442             return Feature.OBSERVATION();
1443             //            return getFeature("Specimen or observation");
1444         }else{
1445             String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1446             logger.warn(String.format(message, type.getMessage()));
1447             return Feature.OBSERVATION();
1448             //            return getFeature("Specimen or observation");
1449
1450         }
1451     }
1452
1453     private Feature getFeature(String featureName, Abcd206ImportState state){
1454         List<Feature> features = getTermService().list(Feature.class, null,null,null,null);
1455         Feature currentFeature=null;
1456         for (Feature feature: features){
1457             String tmpF = feature.getTitleCache();
1458             if (tmpF.equalsIgnoreCase(featureName)) {
1459                 currentFeature=feature;
1460             }
1461         }
1462         if (currentFeature == null) {
1463             currentFeature=Feature.NewInstance(featureName, featureName, featureName);
1464             save(currentFeature, state);
1465         }
1466         return currentFeature;
1467     }
1468
1469     /**
1470      *  getTaxon : search for an existing taxon in the database
1471      * @param state : the ABCD import state
1472      * @param scientificName : the name (string)
1473      * @param i : the current unit position in the abcd file
1474      * @param rank : the rank for the taxon
1475      * @return a Taxon
1476      */
1477     @SuppressWarnings("rawtypes")
1478     private Taxon getTaxon(Abcd206ImportState state, String scientificName, int i, Rank rank) {
1479 //        System.out.println("GETTAXON "+scientificName);
1480         Abcd206ImportConfigurator config = state.getConfig();
1481         Taxon taxon = null;
1482         NonViralName<?> taxonName = null;
1483
1484         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
1485
1486 //        System.out.println("config.isReuseExistingTaxaWhenPossible() :"+config.isReuseExistingTaxaWhenPossible());
1487         if (config.isReuseExistingTaxaWhenPossible()){
1488             List<TaxonBase> c = null;
1489             try {
1490                 List<TaxonBase> taxonbaseList = getTaxonService().listByTitle(Taxon.class, scientificName+" sec", MatchMode.BEGINNING, null, null, null, null, null);
1491                 if (taxonbaseList.size()>0){
1492                     if(config.isInteractWithUser() && config.isAllowReuseOtherClassifications()){
1493                         taxon = sui.askWhereToFixData(scientificName,taxonbaseList, classification);
1494                     } else {
1495                         taxon = sui.lookForTaxaIntoCurrentClassification(taxonbaseList, classification);
1496                     }
1497                 }
1498                 else{
1499                     c = getTaxonService().searchTaxaByName(scientificName, ref);
1500                     if(config.isInteractWithUser() && config.isAllowReuseOtherClassifications()){
1501                         taxon = sui.askWhereToFixData(scientificName,c, classification);
1502                     }
1503                     else{
1504                         taxon = sui.lookForTaxaIntoCurrentClassification(c, classification);
1505                     }
1506                 }
1507             } catch (Exception e) {
1508                 logger.info("Searchtaxabyname failed" + e);
1509                 taxon = null;
1510             }
1511         }
1512         if (!config.isReuseExistingTaxaWhenPossible() || taxon == null){
1513 //            System.out.println("create new taxonName instance "+i+", "+config.isParseNameAutomatically());
1514             if (config.isParseNameAutomatically()){
1515                 taxonName = parseScientificName(scientificName);
1516             }
1517             else{
1518                 if (i>=0 && (dataHolder.atomisedIdentificationList != null || dataHolder.atomisedIdentificationList.size() > 0)) {
1519                     taxonName = setTaxonNameByType(dataHolder.atomisedIdentificationList.get(i), scientificName);
1520                 } else {
1521                     taxonName=null;
1522                 }
1523             }
1524             //            if (taxonName != null) {
1525             //                System.out.println(taxonName.getTitleCache());
1526             //            } else {
1527             //                System.out.println("taxonname: "+taxonName);
1528             //            }
1529             if(taxonName == null){
1530                 taxonName = NonViralName.NewInstance(rank);
1531                 taxonName.setFullTitleCache(scientificName,true);
1532                 taxonName.setTitleCache(scientificName, true);
1533             }
1534 //            System.out.println("ADD NEW TAXON *"+taxonName.getRank()+"*"+taxonName.getTitleCache());
1535             if (rank != null && (taxonName.getRank() ==null || taxonName.getRank().toString().trim().isEmpty())) {
1536                 taxonName.setRank(rank);
1537             }
1538             save(taxonName, state);
1539             taxon = Taxon.NewInstance(taxonName, ref); //sec set null
1540             save(taxon, state);
1541         }
1542         return taxon;
1543     }
1544
1545     /**
1546      * HandleIdentifications : get the scientific names present in the ABCD
1547      * document and store link them with the observation/specimen data
1548      * @param state: the current ABCD import state
1549      * @param derivedUnitFacade : the current derivedunitfacade
1550      */
1551     private void handleIdentifications(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
1552         Abcd206ImportConfigurator config = state.getConfig();
1553
1554         Taxon taxon = null;
1555
1556         String scientificName = "";
1557         boolean preferredFlag = false;
1558
1559         if (dataHolder.nomenclatureCode == ""){
1560             dataHolder.nomenclatureCode = config.getNomenclaturalCode().toString();
1561         }
1562
1563         for (int i = 0; i < dataHolder.identificationList.size(); i++) {
1564             Identification identification = dataHolder.identificationList.get(i);
1565             scientificName = identification.getScientificName().replaceAll(" et ", " & ");
1566
1567             String preferred = identification.getPreferred();
1568             if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || dataHolder.identificationList.size()==1) {
1569                 preferredFlag = true;
1570             }
1571             else {
1572                 preferredFlag = false;
1573             }
1574
1575             if (identification.getCode().indexOf(':') != -1) {
1576                 dataHolder.nomenclatureCode = identification.getCode().split(COLON)[1];
1577             }
1578             else{
1579                 dataHolder.nomenclatureCode = identification.getCode();
1580             }
1581             taxon = getTaxon(state, scientificName, i,null);
1582             addTaxonNode(taxon, state,preferredFlag);
1583             linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
1584         }
1585     }
1586
1587     /**
1588      * @param taxon : a taxon to add as a node
1589      * @param state : the ABCD import state
1590      */
1591     private void addTaxonNode(Taxon taxon, Abcd206ImportState state, boolean preferredFlag) {
1592         logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1593         boolean exist = false;
1594         for (TaxonNode p : classification.getAllNodes()){
1595             try{
1596                 if(p.getTaxon().equals(taxon)) {
1597                     exist =true;
1598                 }
1599             }
1600             catch(Exception e){
1601                 logger.warn("TaxonNode doesn't seem to have a taxon");
1602             }
1603         }
1604         if (!exist){
1605             addParentTaxon(taxon, state, preferredFlag);
1606         }
1607     }
1608
1609     private boolean hasTaxonNodeInClassification(Taxon taxon){
1610         for (TaxonNode node : taxon.getTaxonNodes()){
1611             if(node.getClassification().equals(classification)){
1612                 return true;
1613             }
1614         }
1615         return false;
1616     }
1617
1618     /**
1619      * Add the hierarchy for a Taxon(add higher taxa)
1620      * @param taxon: a taxon to add as a node
1621      * @param state: the ABCD import state
1622      */
1623     private void addParentTaxon(Taxon taxon, Abcd206ImportState state, boolean preferredFlag){
1624         NonViralName<?>  nvname = CdmBase.deproxy(taxon.getName(), NonViralName.class);
1625         Rank rank = nvname.getRank();
1626         Taxon genus =null;
1627         Taxon subgenus =null;
1628         Taxon species = null;
1629         Taxon subspecies = null;
1630         Taxon parent = null;
1631         if (rank.isLower(Rank.GENUS() )){
1632             String prefix = nvname.getGenusOrUninomial();
1633             genus = getTaxon(state, prefix, -1, Rank.GENUS());
1634             if (preferredFlag) {
1635                 parent = saveOrUpdateClassification(null, genus, state);
1636             }
1637
1638         }
1639         if (rank.isLower(Rank.SUBGENUS())){
1640             String prefix = nvname.getGenusOrUninomial();
1641             String name = nvname.getInfraGenericEpithet();
1642             if (name != null){
1643                 subgenus = getTaxon(state, prefix+" "+name, -1, Rank.SUBGENUS());
1644                 if (preferredFlag) {
1645                     parent = saveOrUpdateClassification(genus, subgenus, state);
1646                 }            }
1647         }
1648         if (rank.isLower(Rank.SPECIES())){
1649             if (subgenus!=null){
1650                 String prefix = nvname.getGenusOrUninomial();
1651                 String name = nvname.getInfraGenericEpithet();
1652                 String spe = nvname.getSpecificEpithet();
1653                 if (spe != null){
1654                     species = getTaxon(state, prefix+" "+name+" "+spe, -1, Rank.SPECIES());
1655                     if (preferredFlag) {
1656                         parent =        saveOrUpdateClassification(subgenus, species, state);
1657                     }
1658                 }
1659             }
1660             else{
1661                 String prefix = nvname.getGenusOrUninomial();
1662                 String name = nvname.getSpecificEpithet();
1663                 if (name != null){
1664                     species = getTaxon(state, prefix+" "+name, -1, Rank.SPECIES());
1665                     if (preferredFlag) {
1666                         parent =        saveOrUpdateClassification(genus, species, state);
1667                     }
1668                 }
1669             }
1670         }
1671         if (rank.isInfraSpecific()){
1672             subspecies = getTaxon(state, nvname.getFullTitleCache(), -1, Rank.SUBSPECIES());
1673             if (preferredFlag) {
1674                 parent =        saveOrUpdateClassification(species, subspecies, state);
1675             }
1676         }
1677         if (preferredFlag) {
1678             saveOrUpdateClassification(parent, taxon, state);
1679         }
1680     }
1681
1682     /**
1683      * Link a parent to a child and save it in the current classification
1684      * @param parent: the higher Taxon
1685      * @param child : the lower (or current) Taxon
1686      * return the Taxon from the new created Node
1687      * @param state
1688      */
1689     private Taxon saveOrUpdateClassification(Taxon parent, Taxon child, Abcd206ImportState state) {
1690         TaxonNode node =null;
1691         if (parent != null) {
1692             parent = (Taxon) getTaxonService().find(parent.getUuid());
1693             child = (Taxon) getTaxonService().find(child.getUuid());
1694             //here we do not have to check if the taxon nodes already exists
1695             //this is done by classification.addParentChild()
1696             node = classification.addParentChild(parent, child, ref, "");
1697         }
1698         else {
1699             child = (Taxon) getTaxonService().find(child.getUuid());
1700             //do not add child node if it already exists
1701             if(hasTaxonNodeInClassification(child)){
1702                 return child;
1703             }
1704             node =classification.addChildTaxon(child, ref, null);
1705         }
1706         save(classification, state);
1707         return node.getTaxon();
1708     }
1709
1710     /**
1711      * Parse automatically the scientific name
1712      * @param scientificName: the scientific name to parse
1713      * @return a parsed name
1714      */
1715     private NonViralName<?> parseScientificName(String scientificName) {
1716         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1717         NonViralName<?> taxonName = null;
1718         boolean problem = false;
1719
1720         if(DEBUG){
1721             logger.info("parseScientificName " + dataHolder.nomenclatureCode.toString());
1722         }
1723
1724         if (dataHolder.nomenclatureCode.toString().equals("Zoological") || dataHolder.nomenclatureCode.toString().contains("ICZN")) {
1725             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
1726             if (taxonName.hasProblem()) {
1727                 problem = true;
1728             }
1729         }
1730         if (dataHolder.nomenclatureCode.toString().equals("Botanical") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1731             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
1732             if (taxonName.hasProblem()) {
1733                 problem = true;
1734             }
1735         }
1736         if (dataHolder.nomenclatureCode.toString().equals("Bacterial") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1737             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
1738             if (taxonName.hasProblem()) {
1739                 problem = true;
1740             }
1741         }
1742         if (dataHolder.nomenclatureCode.toString().equals("Cultivar") || dataHolder.nomenclatureCode.toString().contains("ICNCP")) {
1743             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
1744             if (taxonName.hasProblem()) {
1745                 problem = true;
1746             }
1747         }
1748         if (problem) {
1749             logger.info("Parsing with problem in parseScientificName " + scientificName);
1750             return null;
1751         }
1752         return taxonName;
1753
1754     }
1755
1756     /**
1757      * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
1758      * The name is built upon the ABCD fields
1759      * @param atomisedMap : the ABCD atomised fields
1760      * @param fullName : the full scientific name
1761      * @return the corresponding Botanical or Zoological or... name
1762      */
1763     private NonViralName<?> setTaxonNameByType(
1764             HashMap<String, String> atomisedMap, String fullName) {
1765         boolean problem = false;
1766         if(DEBUG) {
1767             logger.info("settaxonnamebytype " + dataHolder.nomenclatureCode.toString());
1768         }
1769
1770         if (dataHolder.nomenclatureCode.equals("Zoological")) {
1771             NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
1772             taxonName.setFullTitleCache(fullName, true);
1773             taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1774             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1775             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
1776             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
1777
1778             if (taxonName.getGenusOrUninomial() != null){
1779                 taxonName.setRank(Rank.GENUS());
1780             }
1781
1782             if (taxonName.getInfraGenericEpithet() != null){
1783                 taxonName.setRank(Rank.SUBGENUS());
1784             }
1785
1786             if (taxonName.getSpecificEpithet() != null){
1787                 taxonName.setRank(Rank.SPECIES());
1788             }
1789
1790             if (taxonName.getInfraSpecificEpithet() != null){
1791                 taxonName.setRank(Rank.SUBSPECIES());
1792             }
1793
1794             Team team = null;
1795             if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1796                 team = Team.NewInstance();
1797                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1798             }
1799             else {
1800                 if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1801                     team = Team.NewInstance();
1802                     team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1803                 }
1804             }
1805             if (team != null) {
1806                 taxonName.setBasionymAuthorTeam(team);
1807             }
1808             else {
1809                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1810                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1811                 }
1812                 else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1813                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
1814                 }
1815             }
1816             if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1817                 team = Team.NewInstance();
1818                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1819                 taxonName.setCombinationAuthorTeam(team);
1820             }
1821             if (taxonName.hasProblem()) {
1822                 logger.info("pb ICZN");
1823                 problem = true;
1824             }
1825             else {
1826                 return taxonName;
1827             }
1828         }
1829         else if (dataHolder.nomenclatureCode.equals("Botanical")) {
1830             BotanicalName taxonName = (BotanicalName) parseScientificName(fullName);
1831             if (taxonName != null){
1832                 return taxonName;
1833             }
1834             else{
1835                 taxonName = BotanicalName.NewInstance(null);
1836             }
1837             taxonName.setFullTitleCache(fullName, true);
1838             taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1839             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
1840             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
1841             try {
1842                 taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
1843             } catch (Exception e) {
1844                 if (taxonName.getGenusOrUninomial() != null){
1845                     taxonName.setRank(Rank.GENUS());
1846                 }
1847                 else if (taxonName.getInfraGenericEpithet() != null){
1848                     taxonName.setRank(Rank.SUBGENUS());
1849                 }
1850                 else if (taxonName.getSpecificEpithet() != null){
1851                     taxonName.setRank(Rank.SPECIES());
1852                 }
1853                 else if (taxonName.getInfraSpecificEpithet() != null){
1854                     taxonName.setRank(Rank.SUBSPECIES());
1855                 }
1856             }
1857             Team team = null;
1858             if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1859                 team = Team.NewInstance();
1860                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1861                 taxonName.setBasionymAuthorTeam(team);
1862             }
1863             if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1864                 team = Team.NewInstance();
1865                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
1866                 taxonName.setCombinationAuthorTeam(team);
1867             }
1868             if (team == null) {
1869                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1870                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1871                 }
1872                 else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1873                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
1874                 }
1875             }
1876             if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1877                 team = Team.NewInstance();
1878                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1879                 taxonName.setCombinationAuthorTeam(team);
1880             }
1881             if (taxonName.hasProblem()) {
1882                 logger.info("pb ICBN");
1883                 problem = true;
1884             }
1885             else {
1886                 return taxonName;
1887             }
1888         }
1889         else if (dataHolder.nomenclatureCode.equals("Bacterial")) {
1890             NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
1891             taxonName.setFullTitleCache(fullName, true);
1892             taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
1893             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1894             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
1895             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
1896
1897             if (taxonName.getGenusOrUninomial() != null){
1898                 taxonName.setRank(Rank.GENUS());
1899             }
1900             else if (taxonName.getInfraGenericEpithet() != null){
1901                 taxonName.setRank(Rank.SUBGENUS());
1902             }
1903             else if (taxonName.getSpecificEpithet() != null){
1904                 taxonName.setRank(Rank.SPECIES());
1905             }
1906             else if (taxonName.getInfraSpecificEpithet() != null){
1907                 taxonName.setRank(Rank.SUBSPECIES());
1908             }
1909
1910             if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1911                 Team team = Team.NewInstance();
1912                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1913                 taxonName.setCombinationAuthorTeam(team);
1914             }
1915             if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
1916                 Team team = Team.NewInstance();
1917                 team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
1918                 taxonName.setBasionymAuthorTeam(team);
1919             }
1920             if (taxonName.hasProblem()) {
1921                 logger.info("pb ICNB");
1922                 problem = true;
1923             }
1924             else {
1925                 return taxonName;
1926             }
1927         }
1928         else if (dataHolder.nomenclatureCode.equals("Cultivar")) {
1929             CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
1930
1931             if (taxonName.hasProblem()) {
1932                 logger.info("pb ICNCP");
1933                 problem = true;
1934             }
1935             else {
1936                 return taxonName;
1937             }
1938             return taxonName;
1939         }
1940
1941         if (problem) {
1942             logger.info("Problem im setTaxonNameByType ");
1943             NonViralName<?> taxonName = NonViralName.NewInstance(null);
1944             taxonName.setFullTitleCache(fullName, true);
1945             return taxonName;
1946         }
1947         NonViralName<?> tn = NonViralName.NewInstance(null);
1948         return tn;
1949     }
1950
1951
1952     /**
1953      * Get a formated string from a hashmap
1954      * @param atomisedMap
1955      * @param key
1956      * @return
1957      */
1958     private String getFromMap(HashMap<String, String> atomisedMap, String key) {
1959         String value = null;
1960         if (atomisedMap.containsKey(key)) {
1961             value = atomisedMap.get(key);
1962         }
1963
1964         try {
1965             if (value != null && key.matches(".*Year.*")) {
1966                 value = value.trim();
1967                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
1968                     String tmp = value.split("[0-9]{4}$")[0];
1969                     int year = Integer.parseInt(value.split(tmp)[1]);
1970                     if (year >= 1752) {
1971                         value = tmp;
1972                     }
1973                     else {
1974                         value = null;
1975                     }
1976                 }
1977                 else {
1978                     value = null;
1979                 }
1980             }
1981         }
1982         catch (Exception e) {
1983             value = null;
1984         }
1985         return value;
1986     }
1987
1988     //    private void compareABCDtoCDM(URI urlFileName, List<String> knownElts, Abcd206XMLFieldGetter abcdFieldGetter) {
1989     //        try {
1990     //            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
1991     //            DocumentBuilder constructeur = factory.newDocumentBuilder();
1992     //            URL url = urlFileName.toURL();
1993     //            Object o = url.getContent();
1994     //            InputStream is = (InputStream) o;
1995     //            Document document = constructeur.parse(is);
1996     //            Element root = document.getDocumentElement();
1997     //            abcdFieldGetter.traverse(root);
1998     //        }
1999     //        catch (ParserConfigurationException e){
2000     //            e.printStackTrace();
2001     //        }
2002     //        catch (SAXException e) {
2003     //            e.printStackTrace();
2004     //        }
2005     //        catch (IOException e) {
2006     //            e.printStackTrace();
2007     //        }
2008     //        Set<String> elts = dataHolder.allABCDelements.keySet();
2009     //        Iterator<String> it = elts.iterator();
2010     //        String elt;
2011     //        while (it.hasNext()) {
2012     //            elt = it.next();
2013     //            if (knownElts.indexOf(elt) == -1) {
2014     //                if(DEBUG) {
2015     //                    logger.info("Unmerged ABCD element: " + elt + " - "+ dataHolder.allABCDelements.get(elt));
2016     //                }
2017     //            }
2018     //        }
2019     //    }
2020
2021     /**
2022      * Load the list of names from the ABCD file and save them
2023      * @param state : the current ABCD import state
2024      * @param unitsList : the unit list from the ABCD file
2025      * @param abcdFieldGetter : the ABCD parser
2026      */
2027     private void prepareCollectors(Abcd206ImportState state, NodeList unitsList, Abcd206XMLFieldGetter abcdFieldGetter) {
2028         List<String> collectors = new ArrayList<String>();
2029         List<String> teams = new ArrayList<String>();
2030         List<List<String>> collectorinteams = new ArrayList<List<String>>();
2031
2032         for (int i = 0; i < unitsList.getLength(); i++) {
2033             this.getCollectorsFromXML((Element) unitsList.item(i), abcdFieldGetter);
2034             for (String agent : dataHolder.gatheringAgentList) {
2035                 collectors.add(agent);
2036             }
2037             List<String> tmpTeam = new ArrayList<String>(new HashSet<String>(dataHolder.gatheringTeamList));
2038             if(!tmpTeam.isEmpty()) {
2039                 teams.add(StringUtils.join(tmpTeam.toArray()," & "));
2040             }
2041             for (String agent:tmpTeam) {
2042                 collectors.add(agent);
2043             }
2044         }
2045
2046         List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
2047         List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
2048
2049
2050         //existing teams in DB
2051         Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
2052         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
2053
2054         Set<UUID> uuids = new HashSet<UUID>();
2055         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
2056             uuids.add(hibernateT.getUuid());
2057         }
2058         if (!uuids.isEmpty()){
2059             List<AgentBase> existingTeams = getAgentService().find(uuids);
2060             for (AgentBase<?> existingP:existingTeams){
2061                 titleCacheTeam.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Team.class));
2062             }
2063         }
2064
2065
2066         Map<String,UUID> teamMap = new HashMap<String, UUID>();
2067         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
2068             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
2069         }
2070
2071         //existing persons in DB
2072         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
2073         Map<String,Person> titleCachePerson = new HashMap<String, Person>();
2074         uuids = new HashSet<UUID>();
2075         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
2076             uuids.add(hibernateP.getUuid());
2077         }
2078
2079         if (!uuids.isEmpty()){
2080             List<AgentBase> existingPersons = getAgentService().find(uuids);
2081             for (AgentBase<?> existingP:existingPersons){
2082                 titleCachePerson.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Person.class));
2083             }
2084         }
2085
2086         Map<String,UUID> personMap = new HashMap<String, UUID>();
2087         for (UuidAndTitleCache<Person> person:hiberPersons){
2088             personMap.put(person.getTitleCache(), person.getUuid());
2089         }
2090
2091         java.util.Collection<Person> personToadd = new ArrayList<Person>();
2092         java.util.Collection<Team> teamToAdd = new ArrayList<Team>();
2093
2094         for (String collector:collectorsU){
2095             Person p = Person.NewInstance();
2096             p.setTitleCache(collector,true);
2097             if (!personMap.containsKey(p.getTitleCache())){
2098                 personToadd.add(p);
2099             }
2100         }
2101         for (String team:teamsU){
2102             Team p = Team.NewInstance();
2103             p.setTitleCache(team,true);
2104             if (!teamMap.containsKey(p.getTitleCache())){
2105                 teamToAdd.add(p);
2106             }
2107         }
2108
2109         if(!personToadd.isEmpty()){
2110             for (Person agent: personToadd){
2111                 save(agent, state);
2112                 titleCachePerson.put(agent.getTitleCache(),CdmBase.deproxy(agent, Person.class) );
2113             }
2114         }
2115
2116         Person ptmp ;
2117         Map <String,Integer>teamdone = new HashMap<String, Integer>();
2118         for (List<String> collteam: collectorinteams){
2119             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
2120                 Team team = new Team();
2121                 boolean em =true;
2122                 for (String collector:collteam){
2123                     ptmp = Person.NewInstance();
2124                     ptmp.setTitleCache(collector,true);
2125                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
2126                     team.addTeamMember(p2);
2127                     em=false;
2128                 }
2129                 if (!em) {
2130                     teamToAdd.add(team);
2131                 }
2132                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
2133             }
2134         }
2135
2136         if(!teamToAdd.isEmpty()){
2137             for (Team agent: teamToAdd){
2138                 save(agent, state);
2139                 titleCacheTeam.put(agent.getTitleCache(), CdmBase.deproxy( agent,Team.class) );
2140             }
2141         }
2142
2143         state.getConfig().setTeams(titleCacheTeam);
2144         state.getConfig().setPersons(titleCachePerson);
2145     }
2146
2147     @Override
2148     protected boolean isIgnore(Abcd206ImportState state) {
2149         return false;
2150     }
2151
2152
2153 }