cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.InputStream;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import javax.xml.parsers.DocumentBuilder;
  24 import javax.xml.parsers.DocumentBuilderFactory;
  25
  26 import org.apache.commons.lang.StringUtils;
  27 import org.apache.log4j.Logger;
  28 import org.springframework.stereotype.Component;
  29 import org.w3c.dom.Document;
  30 import org.w3c.dom.Element;
  31 import org.w3c.dom.NodeList;
  32
  33 import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
  34 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  35 import eu.etaxonomy.cdm.common.UriUtils;
  36 import eu.etaxonomy.cdm.io.specimen.SpecimenImportBase;
  37 import eu.etaxonomy.cdm.io.specimen.SpecimenUserInteraction;
  38 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  39 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  40 import eu.etaxonomy.cdm.model.agent.AgentBase;
  41 import eu.etaxonomy.cdm.model.agent.Institution;
  42 import eu.etaxonomy.cdm.model.agent.Person;
  43 import eu.etaxonomy.cdm.model.agent.Team;
  44 import eu.etaxonomy.cdm.model.common.CdmBase;
  45 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  46 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  47 import eu.etaxonomy.cdm.model.common.Language;
  48 //=======
  49 import eu.etaxonomy.cdm.model.common.LanguageString;
  50 //<<<<<<< .courant
  51 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  52 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  53 //>>>>>>> .fusion-droit.r19057
  54 import eu.etaxonomy.cdm.model.common.UuidAndTitleCache;
  55 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  56 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  57 import eu.etaxonomy.cdm.model.description.Feature;
  58 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  59 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  60 import eu.etaxonomy.cdm.model.location.NamedArea;
  61 import eu.etaxonomy.cdm.model.media.Media;
  62 import eu.etaxonomy.cdm.model.name.BacterialName;
  63 import eu.etaxonomy.cdm.model.name.BotanicalName;
  64 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  65 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  66 import eu.etaxonomy.cdm.model.name.NonViralName;
  67 import eu.etaxonomy.cdm.model.name.Rank;
  68 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  69 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  70 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  71 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  72 import eu.etaxonomy.cdm.model.occurrence.Collection;
  73 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  74 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  75 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  76 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  77 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  78 import eu.etaxonomy.cdm.model.reference.Reference;
  79 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  80 import eu.etaxonomy.cdm.model.taxon.Classification;
  81 import eu.etaxonomy.cdm.model.taxon.Taxon;
  82 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  83 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  84 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  85 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  86
  87 /**
  88  * @author p.kelbert
  89  * @created 20.10.2008
  90  */
  91 @Component
  92 public class Abcd206Import extends SpecimenImportBase<Abcd206ImportConfigurator, Abcd206ImportState> {
  93     private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  94
  95
  96     private final boolean DEBUG = true;
  97
  98     private static final String PREFERRED = "_preferred_";
  99     private static final String CODE = "_code_";
 100     private static final String COLON = ":";
 101     private static final String SPLITTER = "--";
 102
 103
 104     private static String prefix = "";
 105
 106     //TODO make all fields ABCD206ImportState variables
 107     private Classification classification = null;
 108     private Reference<?> ref = null;
 109
 110     private Abcd206DataHolder dataHolder;
 111     private DerivedUnit derivedUnitBase;
 112
 113     private List<OriginalSourceBase<?>> associationRefs = new ArrayList<OriginalSourceBase<?>>();
 114     boolean associationSourcesSet=false;
 115     private List<OriginalSourceBase<?>> descriptionRefs = new ArrayList<OriginalSourceBase<?>>();
 116     boolean descriptionSourcesSet=false;
 117     private List<OriginalSourceBase<?>> derivedUnitSources = new ArrayList<OriginalSourceBase<?>>();
 118     boolean derivedUnitSourcesSet=false;
 119
 120     public Abcd206Import() {
 121         super();
 122     }
 123
 124     @Override
 125     protected boolean doCheck(Abcd206ImportState state) {
 126         logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
 127         return true;
 128     }
 129
 130
 131     @Override
 132     @SuppressWarnings("rawtypes")
 133     public void doInvoke(Abcd206ImportState state) {
 134         state.setTx(startTransaction());
 135         logger.info("INVOKE Specimen Import from ABCD2.06 XML ");
 136
 137         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 138
 139         List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
 140
 141         if (state.getConfig().isInteractWithUser()){
 142             Map<String,Reference> refMap = new HashMap<String, Reference>();
 143             for (Reference tree : references) {
 144                 if (! StringUtils.isBlank(tree.getTitleCache())) {
 145                     refMap.put(tree.getTitleCache(),tree);
 146                 }
 147             }
 148             ref = sui.askForReference(refMap);
 149
 150             if (ref == null){
 151                 String cla = sui.createNewReference();
 152                 if (refMap.get(cla)!= null) {
 153                     ref = refMap.get(cla);
 154                 } else {
 155                     ref = ReferenceFactory.newGeneric();
 156                     ref.setTitle(cla);
 157                 }
 158             }
 159             else{
 160                 ref = getReferenceService().find(ref.getUuid());
 161             }
 162         }else{
 163             if (ref==null){
 164                 String name = NB(state.getConfig().getSourceReferenceTitle());
 165                 for (Reference tree : references) {
 166                     if (! StringUtils.isBlank(tree.getTitleCache())) {
 167                         if (tree.getTitleCache().equalsIgnoreCase(name)) {
 168                             ref=tree;
 169                             System.out.println("FIND SAME REFERENCE");
 170                         }
 171                     }
 172                 }
 173                 if (ref == null){
 174                     ref = ReferenceFactory.newGeneric();
 175                     ref.setTitle("ABCD classic");
 176                 }
 177             }
 178         }
 179         save(ref, state);
 180         state.getConfig().setSourceReference(ref);
 181
 182         List<Classification> classificationList = getClassificationService().list(Classification.class, null, null, null, null);
 183         if (state.getConfig().isUseClassification() && state.getConfig().isInteractWithUser()){
 184             Map<String,Classification> classMap = new HashMap<String, Classification>();
 185             for (Classification tree : classificationList) {
 186                 if (! StringUtils.isBlank(tree.getTitleCache())) {
 187                     classMap.put(tree.getTitleCache(),tree);
 188                 }
 189             }
 190             classification = sui.askForClassification(classMap);
 191             if (classification == null){
 192                 String cla = sui.createNewClassification();
 193                 if (classMap.get(cla)!= null) {
 194                     classification = classMap.get(cla);
 195                 } else {
 196                     classification = Classification.NewInstance(cla, ref, Language.DEFAULT());
 197                 }
 198             }
 199             save(classification, state);
 200         }
 201         else{
 202             if (classification == null) {
 203                 String name = NB(state.getConfig().getClassificationName());
 204                 for (Classification classif : classificationList){
 205                     if (classif.getTitleCache().equalsIgnoreCase(name) && classif.getCitation().equals(ref)) {
 206                         classification=classif;
 207                         System.out.println("FIND SAME CLASSIF");
 208                     }
 209                 }
 210                 if (classification == null){
 211                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 212                 }
 213                 //                if (state.getConfig().getClassificationUuid() != null) {
 214                 //                    classification.setUuid(state.getConfig().getClassificationUuid());
 215                 //                }
 216                 save(classification, state);
 217             }
 218         }
 219
 220         InputStream source = state.getConfig().getSource();
 221         NodeList unitsList = getUnitsNodeList(source);
 222
 223         if (unitsList != null) {
 224             String message = "nb units to insert: " + unitsList.getLength();
 225             logger.info(message);
 226             updateProgress(state, message);
 227
 228             dataHolder = new Abcd206DataHolder();
 229
 230             Abcd206XMLFieldGetter abcdFieldGetter = new Abcd206XMLFieldGetter(dataHolder, prefix);
 231
 232             prepareCollectors(state, unitsList, abcdFieldGetter);
 233
 234             for (int i = 0; i < unitsList.getLength(); i++) {
 235                 System.out.println("------------------------------------------------------------------------------------------");
 236                 associationRefs = new ArrayList<OriginalSourceBase<?>>();
 237                 descriptionRefs = new ArrayList<OriginalSourceBase<?>>();
 238                 derivedUnitSources = new ArrayList<OriginalSourceBase<?>>();
 239
 240                 this.setUnitPropertiesXML( (Element) unitsList.item(i), abcdFieldGetter);
 241                 //                              refreshTransaction(state);
 242                 this.handleSingleUnit(state);
 243
 244                 // compare the ABCD elements added in to the CDM and the
 245                 // unhandled ABCD elements
 246                 //compareABCDtoCDM(sourceName, dataHolder.knownABCDelements, abcdFieldGetter);
 247
 248                 // reset the ABCD elements added in CDM
 249                 // knownABCDelements = new ArrayList<String>();
 250                 dataHolder.allABCDelements = new HashMap<String, String>();
 251             }
 252             getReferenceService().deduplicate(Reference.class, null, null);
 253             getClassificationService().deduplicate(Classification.class, null, null);
 254         }
 255         commitTransaction(state.getTx());
 256         return;
 257
 258     }
 259
 260
 261     protected NodeList getUnitsNodeList(URI source) {
 262         try {
 263             InputStream is = UriUtils.getInputStream(source);
 264             return getUnitsNodeList(is);
 265         } catch (Exception e) {
 266             logger.warn(e);
 267             throw new RuntimeException(e);
 268         }
 269     }
 270
 271     /**
 272      * Return the list of root nodes for an ABCD 2.06 XML file
 273      * @param fileName: the file's location
 274      * @return the list of root nodes ("Unit")
 275      */
 276     protected NodeList getUnitsNodeList(InputStream inputStream) {
 277         NodeList unitList = null;
 278         try {
 279             DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 280             DocumentBuilder builder = factory.newDocumentBuilder();
 281
 282             Document document = builder.parse(inputStream);
 283             Element root = document.getDocumentElement();
 284             unitList = root.getElementsByTagName("Unit");
 285             if (unitList.getLength() == 0) {
 286                 unitList = root.getElementsByTagName("abcd:Unit");
 287                 prefix = "abcd:";
 288             }
 289         } catch (Exception e) {
 290             logger.warn(e);
 291         }
 292         return unitList;
 293     }
 294
 295     /**
 296      * Handle a single unit
 297      * @param state
 298      */
 299     @SuppressWarnings("rawtypes")
 300     private void handleSingleUnit(Abcd206ImportState state) {
 301         if (DEBUG) {
 302             logger.info("handleSingleUnit "+ref);
 303         }
 304         try {
 305             updateProgress(state, "Importing data for unit: " + dataHolder.unitID);
 306
 307             // create facade
 308             DerivedUnitFacade derivedUnitFacade = getFacade();
 309             derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 310
 311             /**
 312              * GATHERING EVENT
 313              */
 314             // gathering event
 315             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(getTermService(), dataHolder.locality, dataHolder.languageIso,
 316                     dataHolder.longitude, dataHolder.latitude, dataHolder.gatheringAgentList, dataHolder.gatheringTeamList,state.getConfig());
 317
 318             // country
 319             UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea();
 320             //  unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(), getTermService());
 321             unitsGatheringArea.setParams(dataHolder.isocountry, dataHolder.country, state.getConfig(), getTermService(), getOccurrenceService());
 322
 323             DefinedTermBase<?> areaCountry =  unitsGatheringArea.getCountry();
 324
 325             // other areas
 326             unitsGatheringArea = new UnitsGatheringArea();
 327             //            unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(),getTermService());
 328             unitsGatheringArea.setAreas(dataHolder.namedAreaList,state.getConfig(), getTermService());
 329             ArrayList<DefinedTermBase> nas = unitsGatheringArea.getAreas();
 330             for (DefinedTermBase namedArea : nas) {
 331                 unitsGatheringEvent.addArea(namedArea);
 332             }
 333
 334             // copy gathering event to facade
 335             GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 336             derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 337             derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 338             derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 339             derivedUnitFacade.setCountry((NamedArea)areaCountry);
 340
 341             for(DefinedTermBase<?> area:unitsGatheringArea.getAreas()){
 342                 derivedUnitFacade.addCollectingArea((NamedArea) area);
 343             }
 344             //            derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 345             // TODO exsiccatum
 346
 347             // add fieldNumber
 348             derivedUnitFacade.setFieldNumber(NB(dataHolder.fieldNumber));
 349
 350             // //add Multimedia URLs
 351             if (dataHolder.multimediaObjects.size() != -1) {
 352                 for (String multimediaObject : dataHolder.multimediaObjects) {
 353                     Media media;
 354                     try {
 355                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA, false);
 356                         derivedUnitFacade.addDerivedUnitMedia(media);
 357                     } catch (MalformedURLException e) {
 358                         // TODO Auto-generated catch block
 359                         e.printStackTrace();
 360                     }
 361
 362                 }
 363             }
 364
 365             //                  /*
 366             //                   * merge AND STORE DATA
 367             //                   */
 368             //                  getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 369             //
 370             //                  for (NamedArea area : otherAreas) {
 371             //                          getTermService().saveOrUpdate(area);// merge it sooner (foreach area)
 372             //                  }
 373
 374             save(unitsGatheringEvent.getLocality(), state);
 375
 376             // handle collection data
 377             setCollectionData(state, derivedUnitFacade);
 378
 379             //Reference stuff
 380             SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 381             Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
 382
 383             dataHolder.docSources = new ArrayList<String>();
 384             for (String[] fullReference : dataHolder.referenceList) {
 385                 System.out.println("referenceFullList "+fullReference);
 386                 String strReference=fullReference[0];
 387                 String citationDetail = fullReference[1];
 388                 String citationURL = fullReference[2];
 389
 390                 if (!citationURL.isEmpty()) {
 391                     citationDetail+=", "+citationURL;
 392                 }
 393
 394                 Reference<?> reference = ReferenceFactory.newGeneric();
 395                 reference.setTitleCache(strReference);
 396                 System.out.println("reference has problem?" +reference.hasProblem());
 397
 398
 399                 IdentifiableSource sour = getIdentifiableSource(reference,citationDetail);
 400
 401                 try{
 402                     if (sour.getCitation() != null){
 403                             if(StringUtils.isNotBlank(sour.getCitationMicroReference())) {
 404                                 dataHolder.docSources.add(sour.getCitation().getTitleCache()+ "---"+sour.getCitationMicroReference());
 405                             } else {
 406                                 dataHolder.docSources.add(sour.getCitation().getTitleCache());
 407                             }
 408                     }
 409                 }catch(Exception e){
 410                     logger.warn("oups");
 411                 }
 412                 //TODO what do we need tmp for?
 413                 String tmp = sour.getCitation().getTitleCache();
 414                 if (StringUtils.isNotBlank(sour.getCitationMicroReference())) {
 415                     tmp +=" ("+sour.getCitationMicroReference()+")";
 416                 }
 417                 reference.addSource(sour);
 418                 getReferenceService().saveOrUpdate(reference);
 419             }
 420
 421
 422             List<OriginalSourceBase> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 423             List<OriginalSourceBase> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 424
 425             Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
 426             if(issTmp2!=null) {
 427                 osbSet.addAll(issTmp2);
 428             }
 429             if(issTmp!=null) {
 430                 osbSet.addAll(issTmp);
 431             }
 432
 433
 434             for( OriginalSourceBase<?> osb:osbSet) {
 435                 if(osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
 436                     try{
 437                         sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
 438                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
 439                 } else{
 440                     try{
 441                         sourceMap.put(osb.getCitation().getTitleCache(),osb);
 442                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
 443                 }
 444             }
 445
 446             if( state.getConfig().isInteractWithUser()){
 447                 List<OriginalSourceBase<?>>sources=null;
 448                 if(!derivedUnitSourcesSet){
 449                     sources= sui.askForSource(sourceMap, "the unit itself","",getReferenceService(), dataHolder.docSources);
 450                     derivedUnitSources=sources;
 451                     derivedUnitSourcesSet=true;
 452                 }
 453                 else{
 454                     sources=derivedUnitSources;
 455                 }
 456                 for (OriginalSourceBase<?> sour:sources){
 457                     if(sour.isInstanceOf(IdentifiableSource.class)){
 458                         if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
 459                             derivedUnitFacade.addSource((IdentifiableSource)sour.clone());
 460                         }
 461                     }else{
 462                         if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
 463                             derivedUnitFacade.addSource(OriginalSourceType.Import,sour.getCitation(),sour.getCitationMicroReference(), ioName);
 464                         }
 465                     }
 466                 }
 467             }else{
 468                 for (OriginalSourceBase<?> sr : sourceMap.values()){
 469                     if(sr.isInstanceOf(IdentifiableSource.class)){
 470                         if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
 471                             derivedUnitFacade.addSource((IdentifiableSource)sr.clone());
 472                         }
 473                     }else{
 474                         if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
 475                             derivedUnitFacade.addSource(OriginalSourceType.Import,sr.getCitation(),sr.getCitationMicroReference(), ioName);
 476                         }
 477                     }
 478                 }
 479             }
 480
 481             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 482
 483             //=======
 484             save(derivedUnitBase, state);
 485
 486             // handle identifications
 487             handleIdentifications(state, derivedUnitFacade);
 488
 489             if(DEBUG) {
 490                 logger.info("saved ABCD specimen ...");
 491             }
 492
 493         } catch (Exception e) {
 494             logger.warn("Error when reading record!!");
 495             e.printStackTrace();
 496             state.setUnsuccessfull();
 497         }
 498
 499         return;
 500     }
 501
 502     /**
 503      * @param derivedUnitFacade
 504      * @param sour
 505      * @return
 506      */
 507     private boolean sourceNotLinkedToElement(DerivedUnitFacade derivedUnitFacade, OriginalSourceBase<?> source) {
 508         Set<IdentifiableSource> linkedSources = derivedUnitFacade.getSources();
 509         for (IdentifiableSource is:linkedSources){
 510
 511
 512             if (is.getCitation()!=null && is.getCitation().getTitleCache().equalsIgnoreCase(source.getCitation().getTitleCache())){
 513                 String isDetail =  is.getCitationMicroReference();
 514                 if (StringUtils.isBlank(isDetail) && StringUtils.isBlank(source.getCitationMicroReference())
 515                         || isDetail != null && isDetail.equalsIgnoreCase(source.getCitationMicroReference()) ) {
 516                     return false;
 517                 }
 518             }
 519         }
 520         return true;
 521     }
 522
 523     /**
 524      * @param reference
 525      * @param citationDetail
 526      * @return
 527      */
 528     //FIXME this method is highly critical, because
 529     //  * it will have serious performance and memory problems with large databases
 530     //        (databases may easily have >1 Mio source records)
 531     //  * it does not make sense to search for existing sources and then clone them
 532     //    we need to search for existing references instead and use them (if exist)
 533     //    for our new source.
 534     private IdentifiableSource getIdentifiableSource(Reference<?> reference, String citationDetail) {
 535
 536         List<OriginalSourceBase> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 537
 538
 539         if (reference != null){
 540                 try {
 541                         for (OriginalSourceBase<?> osb: issTmp){
 542                         if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 543                                 String osbDetail = osb.getCitationMicroReference();
 544                                 if (StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail)
 545                                                 || osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail) ) {
 546                                     System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 547                                     return (IdentifiableSource) osb.clone();
 548                                 }
 549                         }
 550                     }
 551                 } catch (CloneNotSupportedException e) {
 552                     throw new RuntimeException(e);
 553                 } catch (Exception e1){
 554                         e1.printStackTrace();
 555                 }
 556         }
 557
 558         IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 559         return sour;
 560     }
 561
 562     /**
 563      * @param reference
 564      * @param citationDetail
 565      * @return
 566      */
 567     private DescriptionElementSource getDescriptionSource(Reference<?> reference, String citationDetail) {
 568
 569         List<OriginalSourceBase> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 570
 571         try {
 572             for (OriginalSourceBase<?> osb:issTmp2){
 573                 if (osb.getCitation().equals(reference) && osb.getCitationMicroReference().equalsIgnoreCase(citationDetail)) {
 574                     return (DescriptionElementSource) osb.clone();
 575                 }
 576             }
 577         } catch (CloneNotSupportedException e) {
 578             // TODO Auto-generated catch block
 579             e.printStackTrace();
 580         }
 581
 582         DescriptionElementSource sour = DescriptionElementSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 583         return sour;
 584     }
 585
 586
 587     /**
 588      * Very fast and dirty implementation to allow handling of transient objects as described in
 589      * https://dev.e-taxonomy.eu/trac/ticket/3726
 590      *
 591      * Not yet complete.
 592      *
 593      * @param cdmBase
 594      * @param state
 595      */
 596     private void save(CdmBase cdmBase, Abcd206ImportState state) {
 597         ICdmApplicationConfiguration cdmRepository = state.getConfig().getCdmAppController();
 598         if (cdmRepository == null){
 599             cdmRepository = this;
 600         }
 601
 602         if (cdmBase.isInstanceOf(LanguageString.class)){
 603             cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 604         }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 605             cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 606         }else if (cdmBase.isInstanceOf(Reference.class)){
 607             cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 608         }else if (cdmBase.isInstanceOf(Classification.class)){
 609             cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 610         }else if (cdmBase.isInstanceOf(AgentBase.class)){
 611             cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 612         }else if (cdmBase.isInstanceOf(Collection.class)){
 613             cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 614         }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 615             cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 616         }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 617             cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 618         }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 619             cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 620         }else{
 621             throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 622         }
 623
 624     }
 625
 626     /**
 627      * setCollectionData : store the collection object into the
 628      * derivedUnitFacade
 629      *
 630      * @param state
 631      */
 632     private void setCollectionData(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
 633         // set catalogue number (unitID)
 634         derivedUnitFacade.setCatalogNumber(NB(dataHolder.unitID));
 635         derivedUnitFacade.setAccessionNumber(NB(dataHolder.accessionNumber));
 636         // derivedUnitFacade.setCollectorsNumber(NB(dataHolder.collectorsNumber));
 637
 638         /*
 639          * INSTITUTION & COLLECTION
 640          */
 641         // manage institution
 642         Institution institution = this.getInstitution(NB(dataHolder.institutionCode), state);
 643         // manage collection
 644         Collection collection = this.getCollection(institution, NB(dataHolder.collectionCode), state);
 645         // link specimen & collection
 646         derivedUnitFacade.setCollection(collection);
 647     }
 648
 649     /**
 650      * getFacade : get the DerivedUnitFacade based on the recordBasis
 651      *
 652      * @return DerivedUnitFacade
 653      */
 654     private DerivedUnitFacade getFacade() {
 655         if(DEBUG) {
 656             logger.info("getFacade()");
 657         }
 658         SpecimenOrObservationType type = null;
 659
 660         // create specimen
 661         if (NB((dataHolder.recordBasis)) != null) {
 662             if (dataHolder.recordBasis.toLowerCase().startsWith("s") || dataHolder.recordBasis.toLowerCase().contains("specimen")) {// specimen
 663                 type = SpecimenOrObservationType.PreservedSpecimen;
 664             }
 665             if (dataHolder.recordBasis.toLowerCase().startsWith("o")) {
 666                 type = SpecimenOrObservationType.Observation;
 667             }
 668             if (dataHolder.recordBasis.toLowerCase().contains("fossil")){
 669                 type = SpecimenOrObservationType.Fossil;
 670             }
 671             if (dataHolder.recordBasis.toLowerCase().startsWith("l")) {
 672                 type = SpecimenOrObservationType.LivingSpecimen;
 673             }
 674             if (type == null) {
 675                 logger.info("The basis of record does not seem to be known: " + dataHolder.recordBasis);
 676                 type = SpecimenOrObservationType.DerivedUnit;
 677             }
 678             // TODO fossils?
 679         } else {
 680             logger.info("The basis of record is null");
 681             type = SpecimenOrObservationType.DerivedUnit;
 682         }
 683         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 684         return derivedUnitFacade;
 685     }
 686
 687     private void getCollectorsFromXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 688         NodeList group;
 689
 690         group = root.getChildNodes();
 691         for (int i = 0; i < group.getLength(); i++) {
 692             if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 693                 group = group.item(i).getChildNodes();
 694                 break;
 695             }
 696         }
 697         dataHolder.gatheringAgentList = new ArrayList<String>();
 698         dataHolder.gatheringTeamList = new ArrayList<String>();
 699         abcdFieldGetter.getType(root);
 700         abcdFieldGetter.getGatheringPeople(root);
 701     }
 702
 703     /**
 704      * Store the unit's properties into variables Look which unit is the
 705      * preferred one Look what kind of name it is supposed to be, for the
 706      * parsing (Botanical, Zoological)
 707      *
 708      * @param racine: the root node for a single unit
 709      */
 710     private void setUnitPropertiesXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 711         try {
 712             NodeList group;
 713
 714             group = root.getChildNodes();
 715             for (int i = 0; i < group.getLength(); i++) {
 716                 if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 717                     group = group.item(i).getChildNodes();
 718                     break;
 719                 }
 720             }
 721             dataHolder.identificationList = new ArrayList<String>();
 722             dataHolder.statusList = new ArrayList<SpecimenTypeDesignationStatus>();
 723             dataHolder.atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
 724             dataHolder.referenceList = new ArrayList<String[]>();
 725             dataHolder.multimediaObjects = new ArrayList<String>();
 726
 727             abcdFieldGetter.getScientificNames(group);
 728             abcdFieldGetter.getType(root);
 729
 730             if(DEBUG) {
 731                 logger.info("this.identificationList "+dataHolder.identificationList.toString());
 732             }
 733             abcdFieldGetter.getIDs(root);
 734             abcdFieldGetter.getRecordBasis(root);
 735             abcdFieldGetter.getMultimedia(root);
 736             abcdFieldGetter.getNumbers(root);
 737             abcdFieldGetter.getGeolocation(root);
 738             abcdFieldGetter.getGatheringPeople(root);
 739             boolean referencefound = abcdFieldGetter.getReferences(root);
 740             if (!referencefound) {
 741                 String[]a = {ref.getTitleCache(),"",""};
 742                 dataHolder.referenceList.add(a);
 743             }
 744
 745         } catch (Exception e) {
 746             logger.info("Error occured while parsing XML file" + e);
 747         }
 748     }
 749
 750     /**
 751      * Look if the Institution does already exist
 752      * @param institutionCode: a string with the institutioncode
 753      * @param config : the configurator
 754      * @return the Institution (existing or new)
 755      */
 756     @SuppressWarnings("rawtypes")
 757     private Institution getInstitution(String institutionCode, Abcd206ImportState state) {
 758         Institution institution=null;
 759         List<AgentBase> institutions;
 760         try {
 761             institutions = getAgentService().list(Institution.class, null, null, null, null);
 762         } catch (Exception e) {
 763             institutions = new ArrayList<AgentBase>();
 764             logger.warn(e);
 765         }
 766         if (institutions.size() > 0 && state.getConfig().isReUseExistingMetadata()) {
 767             for (AgentBase inst:institutions){
 768                 Institution institut = (Institution)inst;
 769                 try{
 770                     if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 771                         institution=institut;
 772                     }
 773                 }catch(Exception e){logger.warn("no institution code in the db");}
 774             }
 775         }
 776         if(DEBUG) {
 777             if(institution !=null) {
 778                 logger.info("getinstitution " + institution.toString());
 779             }
 780         }
 781         if (institution == null){
 782             // create institution
 783             institution = Institution.NewInstance();
 784             institution.setCode(institutionCode);
 785             institution.setTitleCache(institutionCode);
 786         }
 787         save(institution, state);
 788         return institution;
 789     }
 790
 791     /**
 792      * Look if the Collection does already exist
 793      * @param collectionCode
 794      * @param collectionCode: a string
 795      * @param config : the configurator
 796      * @return the Collection (existing or new)
 797      */
 798     private Collection getCollection(Institution institution, String collectionCode, Abcd206ImportState state) {
 799         Collection collection = null;
 800         List<Collection> collections;
 801         try {
 802             collections = getCollectionService().list(Collection.class, null, null, null, null);
 803         } catch (Exception e) {
 804             collections = new ArrayList<Collection>();
 805         }
 806         if (collections.size() > 0 && state.getConfig().isReUseExistingMetadata()) {
 807             for (Collection coll:collections){
 808                 if (coll.getInstitute() != null) {
 809                     if (coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 810                         collection=coll;
 811                     }
 812                 }
 813             }
 814         }
 815
 816         if(collection == null){
 817             collection =Collection.NewInstance();
 818             collection.setCode(collectionCode);
 819             collection.setInstitute(institution);
 820             collection.setTitleCache(collectionCode);
 821         }
 822         save(collection, state);
 823         return collection;
 824     }
 825
 826
 827     /**
 828      * join DeterminationEvent to the Taxon Object
 829      * @param state : the ABCD import state
 830      * @param taxon: the current Taxon
 831      * @param preferredFlag :if the current name is preferred
 832      * @param derivedFacade : the derived Unit Facade
 833      */
 834     @SuppressWarnings("rawtypes")
 835     private void linkDeterminationEvent(Abcd206ImportState state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
 836         Abcd206ImportConfigurator config = state.getConfig();
 837         if(DEBUG){
 838             logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
 839         }
 840
 841         DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 842         determinationEvent.setTaxon(taxon);
 843         determinationEvent.setPreferredFlag(preferredFlag);
 844
 845         determinationEvent.setIdentifiedUnit(derivedUnitBase);
 846         derivedUnitBase.addDetermination(determinationEvent);
 847
 848         try {
 849             if(DEBUG){
 850                 logger.info("NB TYPES INFO: "+ dataHolder.statusList.size());
 851             }
 852             for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : dataHolder.statusList) {
 853                 if (specimenTypeDesignationstatus != null) {
 854                     if(DEBUG){
 855                         logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
 856                     }
 857
 858                     specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) getTermService().find(specimenTypeDesignationstatus.getUuid());
 859                     //Designation
 860                     TaxonNameBase<?,?> name = taxon.getName();
 861                     SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
 862
 863                     designation.setTypeStatus(specimenTypeDesignationstatus);
 864                     designation.setTypeSpecimen(derivedUnitBase);
 865                     name.addTypeDesignation(designation, true);
 866                 }
 867             }
 868         } catch (Exception e) {
 869             logger.warn("PB addding SpecimenType " + e);
 870         }
 871
 872         for (String[] fullReference : dataHolder.referenceList) {
 873             try{
 874                 System.out.println(fullReference);
 875                 List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
 876
 877                 String strReference=fullReference[0];
 878                 String citationDetail = fullReference[1];
 879                 String citationURL = fullReference[2];
 880
 881                 if (isNotBlank(strReference)){
 882                     Reference<?> reference = null;
 883                     for (Reference<?> refe: references) {
 884                         if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
 885                             reference =refe;
 886                         }
 887                     }
 888                     if (reference ==null){
 889                         reference = ReferenceFactory.newGeneric();
 890                         /*<<<<<<< .courant
 891                     reference.setTitleCache(strReference);
 892                     System.out.println("reference hasproblem2 "+reference.hasProblem());
 893                     IdentifiableSource sour = IdentifiableSource.NewInstance(reference,citationDetail);
 894                     getReferenceService().saveOrUpdate(sour.getCitation());
 895 =======*/
 896                         reference.setTitleCache(strReference, true);
 897                         save(reference, state);
 898                     }
 899                     determinationEvent.addReference(reference);
 900                 }
 901             }catch(Exception e){logger.warn("pv getReferenceList "+e);}
 902         }
 903         save(derivedUnitBase, state);
 904
 905         if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations()) {
 906             if(DEBUG){
 907                 logger.info("isDoCreateIndividualsAssociations");
 908             }
 909
 910             makeIndividualsAssociation(state, taxon, determinationEvent);
 911             save(derivedUnitBase, state);
 912         }
 913     }
 914
 915     /**
 916      * create and link each association (specimen, observation..) to the accepted taxon
 917      * @param state : the ABCD import state
 918      * @param taxon: the current Taxon
 919      * @param determinationEvent:the determinationevent
 920      */
 921     @SuppressWarnings("unused")
 922     private void makeIndividualsAssociation(Abcd206ImportState state, Taxon taxon, DeterminationEvent determinationEvent) {
 923         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 924
 925         if (DEBUG) {
 926             System.out.println("MAKE INDIVIDUALS ASSOCIATION");
 927         }
 928
 929         TaxonDescription taxonDescription = null;
 930         Set<TaxonDescription> descriptions= taxon.getDescriptions();
 931         if (state.getConfig().isInteractWithUser()){
 932             taxonDescription = sui.askForDescriptionGroup(descriptions);
 933         } else {
 934             for (TaxonDescription description : descriptions){
 935                 Set<IdentifiableSource> sources =  description.getTaxon().getSources();
 936                 sources.addAll(description.getSources());
 937                 for (IdentifiableSource source:sources){
 938                     if(ref.equals(source.getCitation())) {
 939                         taxonDescription = description;
 940                     }
 941                 }
 942             }
 943         }
 944         if (taxonDescription == null){
 945             taxonDescription = TaxonDescription.NewInstance(taxon, false);
 946             if(sourceNotLinkedToElement(taxonDescription,ref,null)) {
 947                 taxonDescription.addSource(OriginalSourceType.Import, null, null, ref, null);
 948             }
 949             taxon.addDescription(taxonDescription);
 950         }
 951
 952         //PREPARE REFERENCE QUESTIONS
 953
 954         Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
 955
 956         List<OriginalSourceBase> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 957         List<OriginalSourceBase> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 958
 959         Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
 960         if(issTmp2!=null) {
 961             osbSet.addAll(issTmp2);
 962         }
 963         if(issTmp!=null) {
 964             osbSet.addAll(issTmp);
 965         }
 966
 967
 968         for( OriginalSourceBase<?> osb:osbSet) {
 969             if(osb.getCitationMicroReference() !=null && !osb.getCitationMicroReference().isEmpty()) {
 970                 try{
 971                     sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
 972                 }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
 973             } else{
 974                 try{
 975                     sourceMap.put(osb.getCitation().getTitleCache(),osb);
 976                 }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
 977             }
 978         }
 979
 980         if (state.getConfig().isInteractWithUser()){
 981             List<OriginalSourceBase<?>> res = null;
 982             if(!descriptionSourcesSet){
 983                 res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
 984                         "The current reference is "+ref.getTitleCache(),getReferenceService(), dataHolder.docSources);
 985                 descriptionRefs=res;
 986                 descriptionSourcesSet=true;
 987             }
 988             else{
 989                 res=descriptionRefs;
 990             }
 991             if(res !=null) {
 992                 for (OriginalSourceBase<?> sour:res){
 993                     if(sour.isInstanceOf(IdentifiableSource.class)){
 994                         try {
 995                             if(sourceNotLinkedToElement(taxonDescription,sour)) {
 996                                 taxonDescription.addSource((IdentifiableSource)sour.clone());
 997                             }
 998                         } catch (CloneNotSupportedException e) {
 999                             logger.warn("no cloning?");
1000                         }
1001                     }else{
1002                         if(sourceNotLinkedToElement(taxonDescription,sour)) {
1003                             taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1004                         }
1005                     }
1006                 }
1007             }
1008         }
1009         else {
1010             if(sourceNotLinkedToElement(taxonDescription,ref,null)) {
1011             taxonDescription.addSource(OriginalSourceType.Import,null, null, ref, null);
1012         }
1013         }
1014
1015         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1016         Feature feature = makeFeature(derivedUnitBase);
1017         indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
1018         indAssociation.setFeature(feature);
1019
1020         //<<<<<<< .courant
1021         if (state.getConfig().isInteractWithUser()){
1022             sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1023
1024             issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1025             issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1026
1027             osbSet = new HashSet<OriginalSourceBase>();
1028             if(issTmp2!=null) {
1029                 osbSet.addAll(issTmp2);
1030             }
1031             if(issTmp!=null) {
1032                 osbSet.addAll(issTmp);
1033             }
1034
1035
1036             for( OriginalSourceBase<?> osb:osbSet) {
1037                 if(osb.getCitationMicroReference() !=null && !osb.getCitationMicroReference().isEmpty()) {
1038                     try{
1039                         sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1040                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1041                 } else{
1042                     try{
1043                         sourceMap.put(osb.getCitation().getTitleCache(),osb);
1044                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1045                 }
1046             }
1047
1048             List<OriginalSourceBase<?>> sources =null;
1049             if(!associationSourcesSet) {
1050                 sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1051                         getReferenceService(),dataHolder.docSources);
1052                 associationRefs=sources;
1053                 associationSourcesSet=true;
1054             }
1055             else{
1056                 sources=associationRefs;
1057             }
1058             if(sources !=null) {
1059                 for (OriginalSourceBase<?> source: sources) {
1060                     if(source !=null) {
1061                         if(source.isInstanceOf(DescriptionElementSource.class)){
1062                             try {
1063                                 if(sourceNotLinkedToElement(indAssociation,source)) {
1064                                     indAssociation.addSource((DescriptionElementSource)source.clone());
1065                                 }
1066                             } catch (CloneNotSupportedException e) {
1067                                 logger.warn("clone forbidden?");
1068                             }
1069                         }else{
1070                             if(sourceNotLinkedToElement(indAssociation,source)) {
1071                                 indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1072                             }
1073                             try {
1074                                 if(sourceNotLinkedToElement(derivedUnitBase, source)) {
1075                                     derivedUnitBase.addSource((IdentifiableSource) source.clone());
1076                                 }
1077                             } catch (CloneNotSupportedException e) {
1078                                 // TODO Auto-generated catch block
1079                                 e.printStackTrace();
1080                             }
1081                         }
1082
1083                     }
1084                 }
1085             }
1086         }else {
1087             if(sourceNotLinkedToElement(indAssociation,ref,null)) {
1088                 indAssociation.addSource(OriginalSourceType.Import,null, null, ref, null);
1089             }
1090                 if(sourceNotLinkedToElement(derivedUnitBase, ref,null)) {
1091                     derivedUnitBase.addSource(OriginalSourceType.Import,null, null, ref, null);
1092                 }
1093             for (Reference<?> citation : determinationEvent.getReferences()) {
1094                 if(sourceNotLinkedToElement(indAssociation,citation,null))
1095                  {
1096                     indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1097                 }
1098                     if(sourceNotLinkedToElement(derivedUnitBase, ref,null)) {
1099                         derivedUnitBase.addSource(OriginalSourceType.Import,null, null, ref, null);
1100                     }
1101             }
1102         }
1103
1104         taxonDescription.addElement(indAssociation);
1105         taxonDescription.setTaxon(taxon);
1106
1107         save(taxonDescription, state);
1108         save(taxon, state);
1109     }
1110
1111
1112
1113     /**
1114      * @param derivedUnitBase2
1115      * @param ref2
1116      * @param object
1117      * @return
1118      */
1119     private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference<?> b, String d) {
1120         Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1121         for (IdentifiableSource is:linkedSources){
1122             Reference a = is.getCitation();
1123             String c = is.getCitationMicroReference();
1124
1125             boolean refMatch=false;
1126             boolean microMatch=false;
1127
1128             try{
1129                 if (a==null && b==null) {
1130                     refMatch=true;
1131                 }
1132                 if (a!=null && b!=null) {
1133                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1134                         refMatch=true;
1135                     }
1136                 }
1137             }catch(Exception e){}
1138
1139
1140             try{
1141                 if (c==null && d==null) {
1142                     microMatch=true;
1143                 }
1144                 if(c!=null && d!=null) {
1145                     if(c.equalsIgnoreCase(d)) {
1146                         microMatch=true;
1147                     }
1148                 }
1149             }
1150             catch(Exception e){}
1151
1152             if (microMatch && refMatch) {
1153                 return false;
1154             }
1155
1156
1157         }
1158         return true;
1159     }
1160
1161     /**
1162      * @param derivedUnitBase2
1163      * @param source
1164      * @return
1165      */
1166     private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, OriginalSourceBase<?> source) {
1167         Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1168         for (IdentifiableSource is:linkedSources){
1169             Reference a = is.getCitation();
1170             Reference b = source.getCitation();
1171             String c = is.getCitationMicroReference();
1172             String d = source.getCitationMicroReference();
1173
1174             boolean refMatch=false;
1175             boolean microMatch=false;
1176
1177             try{
1178                 if (a==null && b==null) {
1179                     refMatch=true;
1180                 }
1181                 if (a!=null && b!=null) {
1182                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1183                         refMatch=true;
1184                     }
1185                 }
1186             }catch(Exception e){}
1187
1188
1189             try{
1190                 if (c==null && d==null) {
1191                     microMatch=true;
1192                 }
1193                 if(c!=null && d!=null) {
1194                     if(c.equalsIgnoreCase(d)) {
1195                         microMatch=true;
1196                     }
1197                 }
1198             }
1199             catch(Exception e){}
1200
1201             if (microMatch && refMatch) {
1202                 return false;
1203             }
1204
1205
1206         }
1207         return true;
1208     }
1209
1210     /**
1211      * @param indAssociation
1212      * @param ref2
1213      * @param object
1214      * @return
1215      */
1216     private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, Reference<?> a, String d) {
1217         Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1218         for (DescriptionElementSource is:linkedSources){
1219             Reference b = is.getCitation();
1220             String c = is.getCitationMicroReference();
1221
1222             boolean refMatch=false;
1223             boolean microMatch=false;
1224
1225             try{
1226                 if (a==null && b==null) {
1227                     refMatch=true;
1228                 }
1229                 if (a!=null && b!=null) {
1230                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1231                         refMatch=true;
1232                     }
1233                 }
1234             }catch(Exception e){}
1235
1236
1237             try{
1238                 if (c==null && d==null) {
1239                     microMatch=true;
1240                 }
1241                 if(c!=null && d!=null) {
1242                     if(c.equalsIgnoreCase(d)) {
1243                         microMatch=true;
1244                     }
1245                 }
1246             }
1247             catch(Exception e){}
1248
1249             if (microMatch && refMatch) {
1250                 return false;
1251             }
1252         }
1253         return true;
1254     }
1255
1256     /**
1257      * @param taxonDescription
1258      * @param ref2
1259      * @param object
1260      * @return
1261      */
1262     private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, Reference<?> a, String d) {
1263         Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1264         for (IdentifiableSource is:linkedSources){
1265             Reference b = is.getCitation();
1266             String c = is.getCitationMicroReference();
1267
1268             boolean refMatch=false;
1269             boolean microMatch=false;
1270
1271             try{
1272                 if (a==null && b==null) {
1273                     refMatch=true;
1274                 }
1275                 if (a!=null && b!=null) {
1276                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1277                         refMatch=true;
1278                     }
1279                 }
1280             }catch(Exception e){}
1281
1282
1283             try{
1284                 if (c==null && d==null) {
1285                     microMatch=true;
1286                 }
1287                 if(c!=null && d!=null) {
1288                     if(c.equalsIgnoreCase(d)) {
1289                         microMatch=true;
1290                     }
1291                 }
1292             }
1293             catch(Exception e){}
1294
1295             if (microMatch && refMatch) {
1296                 return false;
1297             }
1298         }
1299         return true;
1300     }
1301
1302     /**
1303      * @param indAssociation
1304      * @param source
1305      * @return
1306      */
1307     private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, OriginalSourceBase<?> source) {
1308         Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1309         for (DescriptionElementSource is:linkedSources){
1310             Reference a = is.getCitation();
1311             Reference b = source.getCitation();
1312             String c = is.getCitationMicroReference();
1313             String d = source.getCitationMicroReference();
1314
1315             boolean refMatch=false;
1316             boolean microMatch=false;
1317
1318             try{
1319                 if (a==null && b==null) {
1320                     refMatch=true;
1321                 }
1322                 if (a!=null && b!=null) {
1323                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1324                         refMatch=true;
1325                     }
1326                 }
1327             }catch(Exception e){}
1328
1329
1330             try{
1331                 if (c==null && d==null) {
1332                     microMatch=true;
1333                 }
1334                 if(c!=null && d!=null) {
1335                     if(c.equalsIgnoreCase(d)) {
1336                         microMatch=true;
1337                     }
1338                 }
1339             }
1340             catch(Exception e){}
1341
1342             if (microMatch && refMatch) {
1343                 return false;
1344             }
1345         }
1346         return true;
1347     }
1348
1349     /**
1350      * @param taxonDescription
1351      * @param sour
1352      * @return
1353      */
1354     private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, OriginalSourceBase<?> sour) {
1355         Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1356         for (IdentifiableSource is:linkedSources){
1357             Reference a = is.getCitation();
1358             Reference b = sour.getCitation();
1359             String c = is.getCitationMicroReference();
1360             String d = sour.getCitationMicroReference();
1361
1362             boolean refMatch=false;
1363             boolean microMatch=false;
1364
1365             try{
1366                 if (a==null && b==null) {
1367                     refMatch=true;
1368                 }
1369                 if (a!=null && b!=null) {
1370                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1371                         refMatch=true;
1372                     }
1373                 }
1374             }catch(Exception e){}
1375
1376
1377             try{
1378                 if (c==null && d==null) {
1379                     microMatch=true;
1380                 }
1381                 if(c!=null && d!=null) {
1382                     if(c.equalsIgnoreCase(d)) {
1383                         microMatch=true;
1384                     }
1385                 }
1386             }
1387             catch(Exception e){}
1388
1389             if (microMatch && refMatch) {
1390                 return false;
1391             }
1392
1393
1394         }
1395         return true;
1396     }
1397
1398     /**
1399      * look for the Feature object (FieldObs, Specimen,...)
1400      * @param unit : a specimen or obersvation base
1401      * @return the corresponding Feature
1402      */
1403     private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1404         SpecimenOrObservationType type = unit.getRecordBasis();
1405         if (type.isFeatureObservation()){
1406             return Feature.OBSERVATION();
1407         }else if (type.isFeatureSpecimen()){
1408             return Feature.SPECIMEN();
1409         }else if (type == SpecimenOrObservationType.DerivedUnit){
1410             return Feature.INDIVIDUALS_ASSOCIATION();
1411         }else{
1412             String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1413             logger.warn(String.format(message, type.getMessage()));
1414             return Feature.INDIVIDUALS_ASSOCIATION();
1415         }
1416     }
1417
1418     /**
1419      *  getTaxon : search for an existing taxon in the database
1420      * @param state : the ABCD import state
1421      * @param scientificName : the name (string)
1422      * @param i : the current unit position in the abcd file
1423      * @param rank : the rank for the taxon
1424      * @return a Taxon
1425      */
1426     @SuppressWarnings("rawtypes")
1427     private Taxon getTaxon(Abcd206ImportState state, String scientificName, int i, Rank rank) {
1428         System.out.println("GETTAXON "+scientificName);
1429         Abcd206ImportConfigurator config = state.getConfig();
1430         Taxon taxon = null;
1431         NonViralName<?> taxonName = null;
1432
1433         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
1434
1435         System.out.println("config.isReuseExistingTaxaWhenPossible() :"+config.isReuseExistingTaxaWhenPossible());
1436         if (config.isReuseExistingTaxaWhenPossible()){
1437             List<TaxonBase> c = null;
1438             try {
1439                 List<TaxonBase> taxonbaseList = getTaxonService().listByTitle(Taxon.class, scientificName+" sec", MatchMode.BEGINNING, null, null, null, null, null);
1440                 if (taxonbaseList.size()>0){
1441                     if(config.isInteractWithUser() && config.isAllowReuseOtherClassifications()){
1442                         taxon = sui.askWhereToFixData(scientificName,taxonbaseList, classification);
1443                     } else {
1444                         taxon = sui.lookForTaxaIntoCurrentClassification(taxonbaseList, classification);
1445                     }
1446                 }
1447                 else{
1448                     c = getTaxonService().searchTaxaByName(scientificName, ref);
1449                     if(config.isInteractWithUser() && config.isAllowReuseOtherClassifications()){
1450                         taxon = sui.askWhereToFixData(scientificName,c, classification);
1451                     }
1452                     else{
1453                         taxon = sui.lookForTaxaIntoCurrentClassification(c, classification);
1454                     }
1455                 }
1456             } catch (Exception e) {
1457                 logger.info("Searchtaxabyname failed" + e);
1458                 taxon = null;
1459             }
1460         }
1461         if (!config.isReuseExistingTaxaWhenPossible() || taxon == null){
1462             System.out.println("create new taxonName instance "+i+", "+config.isParseNameAutomatically());
1463             if (config.isParseNameAutomatically()){
1464                 taxonName = parseScientificName(scientificName);
1465             }
1466             else{
1467                 if (i>=0 && (dataHolder.atomisedIdentificationList != null || dataHolder.atomisedIdentificationList.size() > 0)) {
1468                     taxonName = setTaxonNameByType(dataHolder.atomisedIdentificationList.get(i), scientificName);
1469                 } else {
1470                     taxonName=null;
1471                 }
1472             }
1473             //            if (taxonName != null) {
1474             //                System.out.println(taxonName.getTitleCache());
1475             //            } else {
1476             //                System.out.println("taxonname: "+taxonName);
1477             //            }
1478             if(taxonName == null){
1479                 taxonName = NonViralName.NewInstance(rank);
1480                 taxonName.setFullTitleCache(scientificName,true);
1481                 taxonName.setTitleCache(scientificName, true);
1482             }
1483             System.out.println("ADD NEW TAXON *"+taxonName.getRank()+"*"+taxonName.getTitleCache());
1484             if (rank != null && (taxonName.getRank() ==null || taxonName.getRank().toString().trim().isEmpty())) {
1485                 taxonName.setRank(rank);
1486             }
1487             save(taxonName, state);
1488             taxon = Taxon.NewInstance(taxonName, ref); //sec set null
1489             save(taxon, state);
1490         }
1491         return taxon;
1492     }
1493
1494
1495
1496
1497
1498
1499
1500     /**
1501      * HandleIdentifications : get the scientific names present in the ABCD
1502      * document and store link them with the observation/specimen data
1503      * @param state: the current ABCD import state
1504      * @param derivedUnitFacade : the current derivedunitfacade
1505      */
1506     private void handleIdentifications(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
1507         System.out.println("The reference from handleidentification "+ref);
1508         Abcd206ImportConfigurator config = state.getConfig();
1509
1510         String fullScientificNameString;
1511         Taxon taxon = null;
1512         Rank.GENUS();
1513         Rank.FAMILY();
1514
1515         String scientificName = "";
1516         boolean preferredFlag = false;
1517
1518         List<String> scientificNames = new ArrayList<String>();
1519         if (dataHolder.nomenclatureCode == ""){
1520             dataHolder.nomenclatureCode = config.getNomenclaturalCode().toString();
1521         }
1522
1523         for (int i = 0; i < dataHolder.identificationList.size(); i++) {
1524
1525             fullScientificNameString = dataHolder.identificationList.get(i);
1526             fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
1527
1528             if (fullScientificNameString.indexOf(PREFERRED) != -1) {
1529                 scientificName = fullScientificNameString.split(PREFERRED)[0];
1530                 String pTmp = fullScientificNameString.split(PREFERRED)[1].split(CODE)[0];
1531                 if (pTmp.equals("1") || pTmp.toLowerCase().indexOf("true") != -1) {
1532                     preferredFlag = true;
1533                 }
1534                 else {
1535                     preferredFlag = false;
1536                 }
1537             }
1538             else {
1539                 scientificName = fullScientificNameString;
1540             }
1541             if(DEBUG) {
1542                 logger.info("fullscientificname " + fullScientificNameString + ", *" + dataHolder.nomenclatureCode + "*");
1543             }
1544             if (fullScientificNameString.indexOf(CODE) != -1) {
1545                 if (fullScientificNameString.indexOf(':') != -1) {
1546                     dataHolder.nomenclatureCode = fullScientificNameString.split(CODE)[1].split(COLON)[1];
1547                 }
1548                 else{
1549                     dataHolder.nomenclatureCode = fullScientificNameString.split(CODE)[1];
1550                 }
1551             }
1552             scientificNames.add(scientificName+SPLITTER+preferredFlag+SPLITTER+i);
1553         }
1554         for (String name:scientificNames) {
1555             scientificName = name.split(SPLITTER)[0];
1556             String pref = name.split(SPLITTER)[1];
1557             String index = name.split(SPLITTER)[2];
1558             if (pref.equalsIgnoreCase("true") || scientificNames.size()==1) {
1559                 preferredFlag = true;
1560             } else {
1561                 preferredFlag =false;
1562             }
1563             taxon = getTaxon(state, scientificName,Integer.parseInt(index),null);
1564             addTaxonNode(taxon, state,preferredFlag);
1565             linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
1566         }
1567     }
1568
1569     /**
1570      * @param taxon : a taxon to add as a node
1571      * @param state : the ABCD import state
1572      */
1573     private void addTaxonNode(Taxon taxon, Abcd206ImportState state, boolean preferredFlag) {
1574         logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1575         boolean exist = false;
1576         for (TaxonNode p : classification.getAllNodes()){
1577             try{
1578                 if(p.getTaxon().equals(taxon)) {
1579                     exist =true;
1580                 }
1581             }catch(Exception e){logger.warn("TaxonNode does'nt seem to have a taxon");}
1582         }
1583         if (!exist){
1584             addParentTaxon(taxon, state, preferredFlag);
1585         }
1586     }
1587
1588     /**
1589      * Add the hierarchy for a Taxon(add higher taxa)
1590      * @param taxon: a taxon to add as a node
1591      * @param state: the ABCD import state
1592      */
1593     private void addParentTaxon(Taxon taxon, Abcd206ImportState state, boolean preferredFlag){
1594         System.out.println("addParentTaxon " + taxon.getTitleCache());
1595
1596         NonViralName<?>  nvname = CdmBase.deproxy(taxon.getName(), NonViralName.class);
1597         Rank rank = nvname.getRank();
1598         Taxon genus =null;
1599         Taxon subgenus =null;
1600         Taxon species = null;
1601         Taxon subspecies = null;
1602         Taxon parent = null;
1603         if (rank.isLower(Rank.GENUS() )){
1604             String prefix = nvname.getGenusOrUninomial();
1605             genus = getTaxon(state, prefix, -1, Rank.GENUS());
1606             if (preferredFlag) {
1607                 parent = saveOrUpdateClassification(null, genus, state);
1608             }
1609
1610         }
1611         if (rank.isLower(Rank.SUBGENUS())){
1612             String prefix = nvname.getGenusOrUninomial();
1613             String name = nvname.getInfraGenericEpithet();
1614             if (name != null){
1615                 subgenus = getTaxon(state, prefix+" "+name, -1, Rank.SUBGENUS());
1616                 if (preferredFlag) {
1617                     parent = saveOrUpdateClassification(genus, subgenus, state);
1618                 }            }
1619         }
1620         if (rank.isLower(Rank.SPECIES())){
1621             if (subgenus!=null){
1622                 String prefix = nvname.getGenusOrUninomial();
1623                 String name = nvname.getInfraGenericEpithet();
1624                 String spe = nvname.getSpecificEpithet();
1625                 if (spe != null){
1626                     species = getTaxon(state, prefix+" "+name+" "+spe, -1, Rank.SPECIES());
1627                     if (preferredFlag) {
1628                         parent =        saveOrUpdateClassification(subgenus, species, state);
1629                     }
1630                 }
1631             }
1632             else{
1633                 String prefix = nvname.getGenusOrUninomial();
1634                 String name = nvname.getSpecificEpithet();
1635                 if (name != null){
1636                     species = getTaxon(state, prefix+" "+name, -1, Rank.SPECIES());
1637                     if (preferredFlag) {
1638                         parent =        saveOrUpdateClassification(genus, species, state);
1639                     }
1640                 }
1641             }
1642         }
1643         if (rank.isInfraSpecific()){
1644             subspecies = getTaxon(state, nvname.getFullTitleCache(), -1, Rank.SUBSPECIES());
1645             if (preferredFlag) {
1646                 parent =        saveOrUpdateClassification(species, subspecies, state);
1647             }
1648         }
1649         if (preferredFlag) {
1650             saveOrUpdateClassification(parent, taxon, state);
1651         }
1652     }
1653
1654     /**
1655      * Link a parent to a child and save it in the current classification
1656      * @param parent: the higher Taxon
1657      * @param child : the lower (or current) Taxon
1658      * return the Taxon from the new created Node
1659      * @param state
1660      */
1661     private Taxon saveOrUpdateClassification(Taxon parent, Taxon child, Abcd206ImportState state) {
1662         System.out.println("ADD CLASSIFICATION parent child "+parent+"," +child);
1663         TaxonNode node =null;
1664         if (parent != null) {
1665             parent = (Taxon) getTaxonService().find(parent.getUuid());
1666             child = (Taxon) getTaxonService().find(child.getUuid());
1667             node = classification.addParentChild(parent, child, ref, "");
1668         }
1669         if (parent == null) {
1670             child = (Taxon) getTaxonService().find(child.getUuid());
1671             node =classification.addChildTaxon(child, ref, null);
1672         }
1673         save(classification, state);
1674         return node.getTaxon();
1675     }
1676
1677     /**
1678      * Parse automatically the scientific name
1679      * @param scientificName: the scientific name to parse
1680      * @return a parsed name
1681      */
1682     private NonViralName<?> parseScientificName(String scientificName) {
1683         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1684         NonViralName<?> taxonName = null;
1685         boolean problem = false;
1686
1687         if(DEBUG){
1688             logger.info("parseScientificName " + dataHolder.nomenclatureCode.toString());
1689         }
1690
1691         if (dataHolder.nomenclatureCode.toString().equals("Zoological") || dataHolder.nomenclatureCode.toString().contains("ICZN")) {
1692             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
1693             if (taxonName.hasProblem()) {
1694                 problem = true;
1695             }
1696         }
1697         if (dataHolder.nomenclatureCode.toString().equals("Botanical") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1698             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
1699             if (taxonName.hasProblem()) {
1700                 problem = true;
1701             }
1702         }
1703         if (dataHolder.nomenclatureCode.toString().equals("Bacterial") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1704             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
1705             if (taxonName.hasProblem()) {
1706                 problem = true;
1707             }
1708         }
1709         if (dataHolder.nomenclatureCode.toString().equals("Cultivar") || dataHolder.nomenclatureCode.toString().contains("ICNCP")) {
1710             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
1711             if (taxonName.hasProblem()) {
1712                 problem = true;
1713             }
1714         }
1715         if (problem) {
1716             logger.info("Parsing with problem in parseScientificName " + scientificName);
1717             return null;
1718         }
1719         return taxonName;
1720
1721     }
1722
1723     /**
1724      * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
1725      * The name is built upon the ABCD fields
1726      * @param atomisedMap : the ABCD atomised fields
1727      * @param fullName : the full scientific name
1728      * @return the corresponding Botanical or Zoological or... name
1729      */
1730     private NonViralName<?> setTaxonNameByType(
1731             HashMap<String, String> atomisedMap, String fullName) {
1732         boolean problem = false;
1733         if(DEBUG) {
1734             logger.info("settaxonnamebytype " + dataHolder.nomenclatureCode.toString());
1735         }
1736
1737         if (dataHolder.nomenclatureCode.equals("Zoological")) {
1738             NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
1739             taxonName.setFullTitleCache(fullName, true);
1740             taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1741             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1742             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
1743             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
1744
1745             if (taxonName.getGenusOrUninomial() != null){
1746                 taxonName.setRank(Rank.GENUS());
1747             }
1748
1749             if (taxonName.getInfraGenericEpithet() != null){
1750                 taxonName.setRank(Rank.SUBGENUS());
1751             }
1752
1753             if (taxonName.getSpecificEpithet() != null){
1754                 taxonName.setRank(Rank.SPECIES());
1755             }
1756
1757             if (taxonName.getInfraSpecificEpithet() != null){
1758                 taxonName.setRank(Rank.SUBSPECIES());
1759             }
1760
1761             Team team = null;
1762             if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1763                 team = Team.NewInstance();
1764                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1765             }
1766             else {
1767                 if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1768                     team = Team.NewInstance();
1769                     team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1770                 }
1771             }
1772             if (team != null) {
1773                 taxonName.setBasionymAuthorTeam(team);
1774             }
1775             else {
1776                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1777                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1778                 }
1779                 else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1780                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
1781                 }
1782             }
1783             if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1784                 team = Team.NewInstance();
1785                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1786                 taxonName.setCombinationAuthorTeam(team);
1787             }
1788             if (taxonName.hasProblem()) {
1789                 logger.info("pb ICZN");
1790                 problem = true;
1791             }
1792             else {
1793                 return taxonName;
1794             }
1795         }
1796         else if (dataHolder.nomenclatureCode.equals("Botanical")) {
1797             BotanicalName taxonName = (BotanicalName) parseScientificName(fullName);
1798             if (taxonName != null){
1799                 return taxonName;
1800             }
1801             else{
1802                 taxonName = BotanicalName.NewInstance(null);
1803             }
1804             taxonName.setFullTitleCache(fullName, true);
1805             taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1806             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
1807             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
1808             try {
1809                 taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
1810             } catch (Exception e) {
1811                 if (taxonName.getGenusOrUninomial() != null){
1812                     taxonName.setRank(Rank.GENUS());
1813                 }
1814                 else if (taxonName.getInfraGenericEpithet() != null){
1815                     taxonName.setRank(Rank.SUBGENUS());
1816                 }
1817                 else if (taxonName.getSpecificEpithet() != null){
1818                     taxonName.setRank(Rank.SPECIES());
1819                 }
1820                 else if (taxonName.getInfraSpecificEpithet() != null){
1821                     taxonName.setRank(Rank.SUBSPECIES());
1822                 }
1823             }
1824             Team team = null;
1825             if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1826                 team = Team.NewInstance();
1827                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1828                 taxonName.setBasionymAuthorTeam(team);
1829             }
1830             if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1831                 team = Team.NewInstance();
1832                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
1833                 taxonName.setCombinationAuthorTeam(team);
1834             }
1835             if (team == null) {
1836                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1837                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1838                 }
1839                 else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1840                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
1841                 }
1842             }
1843             if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1844                 team = Team.NewInstance();
1845                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1846                 taxonName.setCombinationAuthorTeam(team);
1847             }
1848             if (taxonName.hasProblem()) {
1849                 logger.info("pb ICBN");
1850                 problem = true;
1851             }
1852             else {
1853                 return taxonName;
1854             }
1855         }
1856         else if (dataHolder.nomenclatureCode.equals("Bacterial")) {
1857             NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
1858             taxonName.setFullTitleCache(fullName, true);
1859             taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
1860             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1861             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
1862             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
1863
1864             if (taxonName.getGenusOrUninomial() != null){
1865                 taxonName.setRank(Rank.GENUS());
1866             }
1867             else if (taxonName.getInfraGenericEpithet() != null){
1868                 taxonName.setRank(Rank.SUBGENUS());
1869             }
1870             else if (taxonName.getSpecificEpithet() != null){
1871                 taxonName.setRank(Rank.SPECIES());
1872             }
1873             else if (taxonName.getInfraSpecificEpithet() != null){
1874                 taxonName.setRank(Rank.SUBSPECIES());
1875             }
1876
1877             if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1878                 Team team = Team.NewInstance();
1879                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1880                 taxonName.setCombinationAuthorTeam(team);
1881             }
1882             if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
1883                 Team team = Team.NewInstance();
1884                 team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
1885                 taxonName.setBasionymAuthorTeam(team);
1886             }
1887             if (taxonName.hasProblem()) {
1888                 logger.info("pb ICNB");
1889                 problem = true;
1890             }
1891             else {
1892                 return taxonName;
1893             }
1894         }
1895         else if (dataHolder.nomenclatureCode.equals("Cultivar")) {
1896             CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
1897
1898             if (taxonName.hasProblem()) {
1899                 logger.info("pb ICNCP");
1900                 problem = true;
1901             }
1902             else {
1903                 return taxonName;
1904             }
1905             return taxonName;
1906         }
1907
1908         if (problem) {
1909             logger.info("Problem im setTaxonNameByType ");
1910             NonViralName<?> taxonName = NonViralName.NewInstance(null);
1911             taxonName.setFullTitleCache(fullName, true);
1912             return taxonName;
1913         }
1914         NonViralName<?> tn = NonViralName.NewInstance(null);
1915         return tn;
1916     }
1917
1918
1919     /**
1920      * Get a formated string from a hashmap
1921      * @param atomisedMap
1922      * @param key
1923      * @return
1924      */
1925     private String getFromMap(HashMap<String, String> atomisedMap, String key) {
1926         String value = null;
1927         if (atomisedMap.containsKey(key)) {
1928             value = atomisedMap.get(key);
1929         }
1930
1931         try {
1932             if (value != null && key.matches(".*Year.*")) {
1933                 value = value.trim();
1934                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
1935                     String tmp = value.split("[0-9]{4}$")[0];
1936                     int year = Integer.parseInt(value.split(tmp)[1]);
1937                     if (year >= 1752) {
1938                         value = tmp;
1939                     }
1940                     else {
1941                         value = null;
1942                     }
1943                 }
1944                 else {
1945                     value = null;
1946                 }
1947             }
1948         }
1949         catch (Exception e) {
1950             value = null;
1951         }
1952         return value;
1953     }
1954
1955     //    private void compareABCDtoCDM(URI urlFileName, List<String> knownElts, Abcd206XMLFieldGetter abcdFieldGetter) {
1956     //        try {
1957     //            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
1958     //            DocumentBuilder constructeur = factory.newDocumentBuilder();
1959     //            URL url = urlFileName.toURL();
1960     //            Object o = url.getContent();
1961     //            InputStream is = (InputStream) o;
1962     //            Document document = constructeur.parse(is);
1963     //            Element root = document.getDocumentElement();
1964     //            abcdFieldGetter.traverse(root);
1965     //        }
1966     //        catch (ParserConfigurationException e){
1967     //            e.printStackTrace();
1968     //        }
1969     //        catch (SAXException e) {
1970     //            e.printStackTrace();
1971     //        }
1972     //        catch (IOException e) {
1973     //            e.printStackTrace();
1974     //        }
1975     //        Set<String> elts = dataHolder.allABCDelements.keySet();
1976     //        Iterator<String> it = elts.iterator();
1977     //        String elt;
1978     //        while (it.hasNext()) {
1979     //            elt = it.next();
1980     //            if (knownElts.indexOf(elt) == -1) {
1981     //                if(DEBUG) {
1982     //                    logger.info("Unmerged ABCD element: " + elt + " - "+ dataHolder.allABCDelements.get(elt));
1983     //                }
1984     //            }
1985     //        }
1986     //    }
1987
1988     /**
1989      * Load the list of names from the ABCD file and save them
1990      * @param state : the current ABCD import state
1991      * @param unitsList : the unit list from the ABCD file
1992      * @param abcdFieldGetter : the ABCD parser
1993      */
1994     private void prepareCollectors(Abcd206ImportState state, NodeList unitsList, Abcd206XMLFieldGetter abcdFieldGetter) {
1995         List<String> collectors = new ArrayList<String>();
1996         List<String> teams = new ArrayList<String>();
1997         List<List<String>> collectorinteams = new ArrayList<List<String>>();
1998
1999         for (int i = 0; i < unitsList.getLength(); i++) {
2000             this.getCollectorsFromXML((Element) unitsList.item(i), abcdFieldGetter);
2001             for (String agent : dataHolder.gatheringAgentList) {
2002                 collectors.add(agent);
2003             }
2004             List<String> tmpTeam = new ArrayList<String>(new HashSet<String>(dataHolder.gatheringTeamList));
2005             if(!tmpTeam.isEmpty()) {
2006                 teams.add(StringUtils.join(tmpTeam.toArray()," & "));
2007             }
2008             for (String agent:tmpTeam) {
2009                 collectors.add(agent);
2010             }
2011         }
2012
2013         List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
2014         List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
2015
2016
2017         //existing teams in DB
2018         Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
2019         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
2020
2021         Set<UUID> uuids = new HashSet<UUID>();
2022         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
2023             uuids.add(hibernateT.getUuid());
2024         }
2025         if (!uuids.isEmpty()){
2026             List<AgentBase> existingTeams = getAgentService().find(uuids);
2027             for (AgentBase<?> existingP:existingTeams){
2028                 titleCacheTeam.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Team.class));
2029             }
2030         }
2031
2032
2033         Map<String,UUID> teamMap = new HashMap<String, UUID>();
2034         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
2035             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
2036         }
2037
2038         //existing persons in DB
2039         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
2040         Map<String,Person> titleCachePerson = new HashMap<String, Person>();
2041         uuids = new HashSet<UUID>();
2042         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
2043             uuids.add(hibernateP.getUuid());
2044         }
2045
2046         if (!uuids.isEmpty()){
2047             List<AgentBase> existingPersons = getAgentService().find(uuids);
2048             for (AgentBase<?> existingP:existingPersons){
2049                 titleCachePerson.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Person.class));
2050             }
2051         }
2052
2053         Map<String,UUID> personMap = new HashMap<String, UUID>();
2054         for (UuidAndTitleCache<Person> person:hiberPersons){
2055             personMap.put(person.getTitleCache(), person.getUuid());
2056         }
2057
2058         java.util.Collection<Person> personToadd = new ArrayList<Person>();
2059         java.util.Collection<Team> teamToAdd = new ArrayList<Team>();
2060
2061         for (String collector:collectorsU){
2062             Person p = Person.NewInstance();
2063             p.setTitleCache(collector,true);
2064             if (!personMap.containsKey(p.getTitleCache())){
2065                 personToadd.add(p);
2066             }
2067         }
2068         for (String team:teamsU){
2069             Team p = Team.NewInstance();
2070             p.setTitleCache(team,true);
2071             if (!teamMap.containsKey(p.getTitleCache())){
2072                 teamToAdd.add(p);
2073             }
2074         }
2075
2076         if(!personToadd.isEmpty()){
2077             for (Person agent: personToadd){
2078                 save(agent, state);
2079                 titleCachePerson.put(agent.getTitleCache(),CdmBase.deproxy(agent, Person.class) );
2080             }
2081         }
2082
2083         Person ptmp ;
2084         Map <String,Integer>teamdone = new HashMap<String, Integer>();
2085         for (List<String> collteam: collectorinteams){
2086             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
2087                 Team team = new Team();
2088                 boolean em =true;
2089                 for (String collector:collteam){
2090                     ptmp = Person.NewInstance();
2091                     ptmp.setTitleCache(collector,true);
2092                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
2093                     team.addTeamMember(p2);
2094                     em=false;
2095                 }
2096                 if (!em) {
2097                     teamToAdd.add(team);
2098                 }
2099                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
2100             }
2101         }
2102
2103         if(!teamToAdd.isEmpty()){
2104             for (Team agent: teamToAdd){
2105                 save(agent, state);
2106                 titleCacheTeam.put(agent.getTitleCache(), CdmBase.deproxy( agent,Team.class) );
2107             }
2108         }
2109
2110         state.getConfig().setTeams(titleCacheTeam);
2111         state.getConfig().setPersons(titleCachePerson);
2112     }
2113
2114     @Override
2115     protected boolean isIgnore(Abcd206ImportState state) {
2116         return false;
2117     }
2118
2119
2120 }