cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.InputStream;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import javax.xml.parsers.DocumentBuilder;
  24 import javax.xml.parsers.DocumentBuilderFactory;
  25
  26 import org.apache.commons.lang.StringUtils;
  27 import org.apache.log4j.Logger;
  28 import org.springframework.stereotype.Component;
  29 import org.w3c.dom.Document;
  30 import org.w3c.dom.Element;
  31 import org.w3c.dom.NodeList;
  32
  33 import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
  34 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  35 import eu.etaxonomy.cdm.common.UriUtils;
  36 import eu.etaxonomy.cdm.io.specimen.SpecimenImportBase;
  37 import eu.etaxonomy.cdm.io.specimen.SpecimenUserInteraction;
  38 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  39 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  40 import eu.etaxonomy.cdm.model.agent.AgentBase;
  41 import eu.etaxonomy.cdm.model.agent.Institution;
  42 import eu.etaxonomy.cdm.model.agent.Person;
  43 import eu.etaxonomy.cdm.model.agent.Team;
  44 import eu.etaxonomy.cdm.model.common.CdmBase;
  45 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  46 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  47 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  48 import eu.etaxonomy.cdm.model.common.Language;
  49 import eu.etaxonomy.cdm.model.common.LanguageString;
  50 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  51 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  52 import eu.etaxonomy.cdm.model.common.UuidAndTitleCache;
  53 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  54 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  55 import eu.etaxonomy.cdm.model.description.Feature;
  56 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  57 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  58 import eu.etaxonomy.cdm.model.location.NamedArea;
  59 import eu.etaxonomy.cdm.model.media.Media;
  60 import eu.etaxonomy.cdm.model.name.BacterialName;
  61 import eu.etaxonomy.cdm.model.name.BotanicalName;
  62 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  63 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  64 import eu.etaxonomy.cdm.model.name.NonViralName;
  65 import eu.etaxonomy.cdm.model.name.Rank;
  66 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  67 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  68 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  69 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  70 import eu.etaxonomy.cdm.model.occurrence.Collection;
  71 import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
  72 import eu.etaxonomy.cdm.model.occurrence.DerivationEventType;
  73 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  74 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  75 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  76 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  77 import eu.etaxonomy.cdm.model.occurrence.MediaSpecimen;
  78 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  79 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  80 import eu.etaxonomy.cdm.model.reference.Reference;
  81 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  82 import eu.etaxonomy.cdm.model.taxon.Classification;
  83 import eu.etaxonomy.cdm.model.taxon.Taxon;
  84 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  85 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  86 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  87 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  88
  89 /**
  90  * @author p.kelbert
  91  * @created 20.10.2008
  92  */
  93 @Component
  94 public class Abcd206Import extends SpecimenImportBase<Abcd206ImportConfigurator, Abcd206ImportState> {
  95     private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  96
  97     private final boolean DEBUG = true;
  98
  99     private static final String PREFERRED = "_preferred_";
 100     private static final String CODE = "_code_";
 101     private static final String COLON = ":";
 102     private static final String SPLITTER = "--";
 103     private static String prefix = "";
 104
 105     //TODO make all fields ABCD206ImportState variables
 106     private Classification classification = null;
 107     private Reference<?> ref = null;
 108
 109     private Abcd206DataHolder dataHolder;
 110     private DerivedUnit derivedUnitBase;
 111     private FieldUnit fieldUnit;
 112
 113     private List<OriginalSourceBase<?>> associationRefs = new ArrayList<OriginalSourceBase<?>>();
 114     boolean associationSourcesSet=false;
 115     private List<OriginalSourceBase<?>> descriptionRefs = new ArrayList<OriginalSourceBase<?>>();
 116     boolean descriptionSourcesSet=false;
 117     private List<OriginalSourceBase<?>> derivedUnitSources = new ArrayList<OriginalSourceBase<?>>();
 118     boolean derivedUnitSourcesSet=false;
 119     private boolean descriptionGroupSet = false;
 120     private TaxonDescription descriptionGroup = null;
 121
 122     public Abcd206Import() {
 123         super();
 124     }
 125
 126     /**
 127      * TODO: quick and dirty fix to avoid NonUniqueObjectExceptions
 128      * They occurred because this import is a bean therefore a singleton but uses
 129      * class variables which should be unique for every single import.
 130      */
 131     private void resetFields(){
 132         classification = null;
 133         ref = null;
 134
 135         dataHolder = null;
 136         derivedUnitBase = null;
 137         fieldUnit = null;
 138
 139         associationRefs = new ArrayList<OriginalSourceBase<?>>();
 140         associationSourcesSet=false;
 141         descriptionRefs = new ArrayList<OriginalSourceBase<?>>();
 142         descriptionSourcesSet=false;
 143         derivedUnitSources = new ArrayList<OriginalSourceBase<?>>();
 144         derivedUnitSourcesSet=false;
 145         descriptionGroupSet = false;
 146         descriptionGroup = null;
 147     }
 148
 149     @Override
 150     protected boolean doCheck(Abcd206ImportState state) {
 151         logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
 152         return true;
 153     }
 154
 155
 156     @Override
 157     @SuppressWarnings("rawtypes")
 158     public void doInvoke(Abcd206ImportState state) {
 159         resetFields();
 160
 161         state.setTx(startTransaction());
 162         logger.info("INVOKE Specimen Import from ABCD2.06 XML ");
 163
 164         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 165
 166         List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
 167
 168         if (state.getConfig().isInteractWithUser()){
 169             Map<String,Reference> refMap = new HashMap<String, Reference>();
 170             for (Reference reference : references) {
 171                 if (! StringUtils.isBlank(reference.getTitleCache())) {
 172                     refMap.put(reference.getTitleCache(),reference);
 173                 }
 174             }
 175             ref = sui.askForReference(refMap);
 176
 177             if (ref == null){
 178                 String cla = sui.createNewReference();
 179                 if (refMap.get(cla)!= null) {
 180                     ref = refMap.get(cla);
 181                 } else {
 182                     ref = ReferenceFactory.newGeneric();
 183                     ref.setTitle(cla);
 184                 }
 185             }
 186             else{
 187                 ref = getReferenceService().find(ref.getUuid());
 188             }
 189         }else{
 190             if (ref==null){
 191                 String name = NB(state.getConfig().getSourceReferenceTitle());
 192                 for (Reference reference : references) {
 193                     if (! StringUtils.isBlank(reference.getTitleCache())) {
 194                         if (reference.getTitleCache().equalsIgnoreCase(name)) {
 195                             ref=reference;
 196 //                            System.out.println("FIND SAME REFERENCE");
 197                         }
 198                     }
 199                 }
 200                 if (ref == null){
 201                     ref = ReferenceFactory.newGeneric();
 202                     ref.setTitle("ABCD classic");
 203                 }
 204             }
 205         }
 206         save(ref, state);
 207         state.getConfig().setSourceReference(ref);
 208
 209         if(state.getConfig().getClassificationUuid()!=null){
 210             //load classification from config if it exists
 211             classification = getClassificationService().load(state.getConfig().getClassificationUuid());
 212         }
 213         if(classification==null){//no existing classification was set in config
 214             List<Classification> classificationList = getClassificationService().list(Classification.class, null, null, null, null);
 215             //get classification via user interaction
 216             if (state.getConfig().isUseClassification() && state.getConfig().isInteractWithUser()){
 217                 Map<String,Classification> classMap = new HashMap<String, Classification>();
 218                 for (Classification tree : classificationList) {
 219                     if (! StringUtils.isBlank(tree.getTitleCache())) {
 220                         classMap.put(tree.getTitleCache(),tree);
 221                     }
 222                 }
 223                 classification = sui.askForClassification(classMap);
 224                 if (classification == null){
 225                     String cla = sui.createNewClassification();
 226                     if (classMap.get(cla)!= null) {
 227                         classification = classMap.get(cla);
 228                     } else {
 229                         classification = Classification.NewInstance(cla, ref, Language.DEFAULT());
 230                     }
 231                 }
 232                 save(classification, state);
 233             }
 234             // create default classification
 235             if (classification == null) {
 236                 String name = NB(state.getConfig().getClassificationName());
 237                 for (Classification classif : classificationList){
 238                     if (classif.getTitleCache().equalsIgnoreCase(name) && classif.getCitation().equals(ref)) {
 239                         classification=classif;
 240                         //                        System.out.println("FIND SAME CLASSIF");
 241                     }
 242                 }
 243                 if (classification == null){
 244                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 245                 }
 246                 //                if (state.getConfig().getClassificationUuid() != null) {
 247                 //                    classification.setUuid(state.getConfig().getClassificationUuid());
 248                 //                }
 249                 save(classification, state);
 250             }
 251         }
 252
 253         InputStream source = state.getConfig().getSource();
 254         NodeList unitsList = getUnitsNodeList(source);
 255
 256         if (unitsList != null) {
 257             String message = "nb units to insert: " + unitsList.getLength();
 258             logger.info(message);
 259             updateProgress(state, message);
 260
 261             dataHolder = new Abcd206DataHolder();
 262
 263             Abcd206XMLFieldGetter abcdFieldGetter = new Abcd206XMLFieldGetter(dataHolder, prefix);
 264
 265             prepareCollectors(state, unitsList, abcdFieldGetter);
 266
 267             associationRefs = new ArrayList<OriginalSourceBase<?>>();
 268             descriptionRefs = new ArrayList<OriginalSourceBase<?>>();
 269             derivedUnitSources = new ArrayList<OriginalSourceBase<?>>();
 270
 271             for (int i = 0; i < unitsList.getLength(); i++) {
 272                 System.out.println("------------------------------------------------------------------------------------------");
 273
 274                 this.setUnitPropertiesXML( (Element) unitsList.item(i), abcdFieldGetter);
 275                 //                              refreshTransaction(state);
 276                 this.handleSingleUnit(state);
 277
 278                 // compare the ABCD elements added in to the CDM and the
 279                 // unhandled ABCD elements
 280                 //compareABCDtoCDM(sourceName, dataHolder.knownABCDelements, abcdFieldGetter);
 281
 282                 // reset the ABCD elements added in CDM
 283                 // knownABCDelements = new ArrayList<String>();
 284                 dataHolder.allABCDelements = new HashMap<String, String>();
 285             }
 286             if(state.getConfig().isDeduplicateReferences()){
 287                 getReferenceService().deduplicate(Reference.class, null, null);
 288             }
 289             if(state.getConfig().isDeduplicateClassifications()){
 290                 getClassificationService().deduplicate(Classification.class, null, null);
 291             }
 292         }
 293         commitTransaction(state.getTx());
 294         return;
 295
 296     }
 297
 298     protected NodeList getUnitsNodeList(URI source) {
 299         try {
 300             InputStream is = UriUtils.getInputStream(source);
 301             return getUnitsNodeList(is);
 302         } catch (Exception e) {
 303             logger.warn(e);
 304             throw new RuntimeException(e);
 305         }
 306     }
 307
 308     /**
 309      * Return the list of root nodes for an ABCD 2.06 XML file
 310      * @param fileName: the file's location
 311      * @return the list of root nodes ("Unit")
 312      */
 313     protected NodeList getUnitsNodeList(InputStream inputStream) {
 314         NodeList unitList = null;
 315         try {
 316             DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 317             DocumentBuilder builder = factory.newDocumentBuilder();
 318
 319             Document document = builder.parse(inputStream);
 320             Element root = document.getDocumentElement();
 321             unitList = root.getElementsByTagName("Unit");
 322             if (unitList.getLength() == 0) {
 323                 unitList = root.getElementsByTagName("abcd:Unit");
 324                 prefix = "abcd:";
 325             }
 326         } catch (Exception e) {
 327             logger.warn(e);
 328         }
 329         return unitList;
 330     }
 331
 332     /**
 333      * Handle a single unit
 334      * @param state
 335      */
 336     @SuppressWarnings("rawtypes")
 337     private void handleSingleUnit(Abcd206ImportState state) {
 338         if (DEBUG) {
 339             logger.info("handleSingleUnit "+ref);
 340         }
 341         try {
 342             updateProgress(state, "Importing data for unit: " + dataHolder.unitID);
 343
 344             // create facade
 345             DerivedUnitFacade derivedUnitFacade = getFacade();
 346             derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 347             fieldUnit = derivedUnitFacade.getFieldUnit(true);
 348
 349             /**
 350              * GATHERING EVENT
 351              */
 352             // gathering event
 353             ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
 354             if(cdmAppController==null){
 355                 cdmAppController = this;
 356             }
 357             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(cdmAppController.getTermService(),
 358                     dataHolder.locality, dataHolder.languageIso, dataHolder.longitude, dataHolder.latitude,
 359                     dataHolder.gatheringElevationText, dataHolder.gatheringElevationMin,
 360                     dataHolder.gatheringElevationMax, dataHolder.gatheringElevationUnit, dataHolder.gatheringDateText,
 361                     dataHolder.gatheringAgentList, dataHolder.gatheringTeamList, state.getConfig());
 362
 363             // country
 364             UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea();
 365             //  unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(), getTermService());
 366             unitsGatheringArea.setParams(dataHolder.isocountry, dataHolder.country, state.getConfig(), cdmAppController.getTermService(), getOccurrenceService());
 367
 368             DefinedTermBase<?> areaCountry =  unitsGatheringArea.getCountry();
 369
 370             // other areas
 371             unitsGatheringArea = new UnitsGatheringArea();
 372             //            unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(),getTermService());
 373             unitsGatheringArea.setAreas(dataHolder.namedAreaList,state.getConfig(), cdmAppController.getTermService());
 374             ArrayList<DefinedTermBase> nas = unitsGatheringArea.getAreas();
 375             for (DefinedTermBase namedArea : nas) {
 376                 unitsGatheringEvent.addArea(namedArea);
 377             }
 378
 379             // copy gathering event to facade
 380             GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 381             derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 382             derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 383             derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 384             derivedUnitFacade.setCountry((NamedArea)areaCountry);
 385             derivedUnitFacade.setAbsoluteElevationText(gatheringEvent.getAbsoluteElevationText());
 386             derivedUnitFacade.setAbsoluteElevation(gatheringEvent.getAbsoluteElevation());
 387             derivedUnitFacade.setAbsoluteElevationMax(gatheringEvent.getAbsoluteElevationMax());
 388             derivedUnitFacade.setGatheringPeriod(gatheringEvent.getTimeperiod());
 389
 390             for(DefinedTermBase<?> area:unitsGatheringArea.getAreas()){
 391                 derivedUnitFacade.addCollectingArea((NamedArea) area);
 392             }
 393             //            derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 394             // TODO exsiccatum
 395
 396             // add fieldNumber
 397             derivedUnitFacade.setFieldNumber(NB(dataHolder.fieldNumber));
 398
 399             // //add Multimedia URLs
 400             if (dataHolder.multimediaObjects.size() != -1) {
 401                 for (String multimediaObject : dataHolder.multimediaObjects) {
 402                     Media media;
 403                     try {
 404                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA);
 405                         derivedUnitFacade.addDerivedUnitMedia(media);
 406                         if(state.getConfig().isAddMediaAsMediaSpecimen()){
 407                             //add media also as specimen scan
 408                             MediaSpecimen mediaSpecimen = MediaSpecimen.NewInstance(SpecimenOrObservationType.Media);
 409                             mediaSpecimen.setMediaSpecimen(media);
 410                             DefinedTermBase specimenScanTerm = getTermService().load(UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03"));
 411                             if(specimenScanTerm instanceof DefinedTerm){
 412                                 mediaSpecimen.setKindOfUnit((DefinedTerm) specimenScanTerm);
 413                             }
 414                             DerivationEvent derivationEvent = DerivationEvent.NewInstance(DerivationEventType.PREPARATION());
 415                             derivationEvent.addDerivative(mediaSpecimen);
 416                             derivedUnitFacade.innerDerivedUnit().addDerivationEvent(derivationEvent);
 417                         }
 418
 419                     } catch (MalformedURLException e) {
 420                         // TODO Auto-generated catch block
 421                         e.printStackTrace();
 422                     }
 423
 424                 }
 425             }
 426
 427             //                  /*
 428             //                   * merge AND STORE DATA
 429             //                   */
 430             //                  getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 431             //
 432             //                  for (NamedArea area : otherAreas) {
 433             //                          getTermService().saveOrUpdate(area);// merge it sooner (foreach area)
 434             //                  }
 435
 436             save(unitsGatheringEvent.getLocality(), state);
 437
 438             // handle collection data
 439             setCollectionData(state, derivedUnitFacade);
 440
 441             //Reference stuff
 442             SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 443             Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
 444
 445             dataHolder.docSources = new ArrayList<String>();
 446             for (String[] fullReference : dataHolder.referenceList) {
 447                 String strReference=fullReference[0];
 448                 String citationDetail = fullReference[1];
 449                 String citationURL = fullReference[2];
 450
 451                 if (!citationURL.isEmpty()) {
 452                     citationDetail+=", "+citationURL;
 453                 }
 454
 455                 Reference<?> reference;
 456                 if(strReference.equals(ref.getTitleCache())){
 457                     reference = ref;
 458                 }
 459                 else{
 460                     reference = ReferenceFactory.newGeneric();
 461                     reference.setTitle(strReference);
 462                 }
 463
 464                 IdentifiableSource sour = getIdentifiableSource(reference,citationDetail);
 465
 466                 try{
 467                     if (sour.getCitation() != null){
 468                         if(StringUtils.isNotBlank(sour.getCitationMicroReference())) {
 469                             dataHolder.docSources.add(sour.getCitation().getTitleCache()+ "---"+sour.getCitationMicroReference());
 470                         } else {
 471                             dataHolder.docSources.add(sour.getCitation().getTitleCache());
 472                         }
 473                     }
 474                 }catch(Exception e){
 475                     logger.warn("oups");
 476                 }
 477                 reference.addSource(sour);
 478                 save(reference, state);
 479             }
 480
 481             List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 482             List<DescriptionElementSource> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 483
 484             Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
 485             if(issTmp2!=null) {
 486                 osbSet.addAll(issTmp2);
 487             }
 488             if(issTmp!=null) {
 489                 osbSet.addAll(issTmp);
 490             }
 491
 492
 493             for( OriginalSourceBase<?> osb:osbSet) {
 494                 if(osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
 495                     try{
 496                         sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
 497                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
 498                 } else{
 499                     try{
 500                         sourceMap.put(osb.getCitation().getTitleCache(),osb);
 501                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
 502                 }
 503             }
 504
 505             if( state.getConfig().isInteractWithUser()){
 506                 List<OriginalSourceBase<?>>sources=null;
 507                 if(!derivedUnitSourcesSet){
 508                     sources= sui.askForSource(sourceMap, "the unit itself","",getReferenceService(), dataHolder.docSources);
 509                     derivedUnitSources=sources;
 510                     derivedUnitSourcesSet=true;
 511                 }
 512                 else{
 513                     sources=derivedUnitSources;
 514                 }
 515 //                System.out.println("nb sources: "+sources.size());
 516 //                System.out.println("derivedunitfacade : "+derivedUnitFacade.getTitleCache());
 517                 for (OriginalSourceBase<?> sour:sources){
 518                     if(sour.isInstanceOf(IdentifiableSource.class)){
 519                         if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
 520 //                            System.out.println("add source to derivedunitfacade1 "+derivedUnitFacade.getTitleCache());
 521                             derivedUnitFacade.addSource((IdentifiableSource)sour.clone());
 522                         }
 523                     }else{
 524                         if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
 525 //                            System.out.println("add source to derivedunitfacade2 "+derivedUnitFacade.getTitleCache());
 526                             derivedUnitFacade.addSource(OriginalSourceType.Import,sour.getCitation(),sour.getCitationMicroReference(), ioName);
 527                         }
 528                     }
 529                 }
 530             }else{
 531                 for (OriginalSourceBase<?> sr : sourceMap.values()){
 532                     if(sr.isInstanceOf(IdentifiableSource.class)){
 533                         if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
 534 //                            System.out.println("add source to derivedunitfacade3 "+derivedUnitFacade.getTitleCache());
 535                             derivedUnitFacade.addSource((IdentifiableSource)sr.clone());
 536                         }
 537                     }else{
 538                         if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
 539 //                            System.out.println("add source to derivedunitfacade4 "+derivedUnitFacade.getTitleCache());
 540                             derivedUnitFacade.addSource(OriginalSourceType.Import,sr.getCitation(),sr.getCitationMicroReference(), ioName);
 541                         }
 542                     }
 543                 }
 544             }
 545
 546             save(derivedUnitBase, state);
 547
 548             // handle identifications
 549             handleIdentifications(state, derivedUnitFacade);
 550
 551             if(DEBUG) {
 552                 logger.info("saved ABCD specimen ...");
 553             }
 554
 555         } catch (Exception e) {
 556             logger.warn("Error when reading record!!");
 557             e.printStackTrace();
 558             state.setUnsuccessfull();
 559         }
 560
 561         return;
 562     }
 563
 564     /**
 565      * @param derivedUnitFacade
 566      * @param sour
 567      * @return
 568      */
 569     private boolean sourceNotLinkedToElement(DerivedUnitFacade derivedUnitFacade, OriginalSourceBase<?> source) {
 570         Set<IdentifiableSource> linkedSources = derivedUnitFacade.getSources();
 571         for (IdentifiableSource is:linkedSources){
 572             if (is.getCitation()!=null && source.getCitation()!=null &&
 573                     is.getCitation().getTitleCache().equalsIgnoreCase(source.getCitation().getTitleCache())){
 574                 String isDetail =  is.getCitationMicroReference();
 575                 if ((StringUtils.isBlank(isDetail) && StringUtils.isBlank(source.getCitationMicroReference()))
 576                         || (isDetail != null && isDetail.equalsIgnoreCase(source.getCitationMicroReference())) ) {
 577                     return false;
 578                 }
 579             }
 580         }
 581         return true;
 582     }
 583
 584     /**
 585      * @param reference
 586      * @param citationDetail
 587      * @return
 588      */
 589     //FIXME this method is highly critical, because
 590     //  * it will have serious performance and memory problems with large databases
 591     //        (databases may easily have >1 Mio source records)
 592     //  * it does not make sense to search for existing sources and then clone them
 593     //    we need to search for existing references instead and use them (if exist)
 594     //    for our new source.
 595     private IdentifiableSource getIdentifiableSource(Reference<?> reference, String citationDetail) {
 596
 597         List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 598
 599
 600         if (reference != null){
 601             try {
 602                 for (OriginalSourceBase<?> osb: issTmp){
 603                     if (osb.getCitation() != null && reference!=null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 604                         String osbDetail = osb.getCitationMicroReference();
 605                         if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 606                                 || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 607 //                            System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 608                             return (IdentifiableSource) osb.clone();
 609                         }
 610                     }
 611                 }
 612             } catch (CloneNotSupportedException e) {
 613                 throw new RuntimeException(e);
 614             } catch (Exception e1){
 615                 e1.printStackTrace();
 616             }
 617         }
 618
 619         IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 620         return sour;
 621     }
 622
 623     //    /**
 624     //     * @param reference
 625     //     * @param citationDetail
 626     //     * @return
 627     //     */
 628     //    private DescriptionElementSource getDescriptionSource(Reference<?> reference, String citationDetail) {
 629     //
 630     //        List<OriginalSourceBase> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 631     //
 632     //        try {
 633     //            for (OriginalSourceBase<?> osb:issTmp2){
 634     //                if (osb.getCitation().equals(reference) && osb.getCitationMicroReference().equalsIgnoreCase(citationDetail)) {
 635     //                    return (DescriptionElementSource) osb.clone();
 636     //                }
 637     //            }
 638     //        } catch (CloneNotSupportedException e) {
 639     //            // TODO Auto-generated catch block
 640     //            e.printStackTrace();
 641     //        }
 642     //
 643     //        DescriptionElementSource sour = DescriptionElementSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 644     //        return sour;
 645     //    }
 646
 647
 648     /**
 649      * Very fast and dirty implementation to allow handling of transient objects as described in
 650      * https://dev.e-taxonomy.eu/trac/ticket/3726
 651      *
 652      * Not yet complete.
 653      *
 654      * @param cdmBase
 655      * @param state
 656      */
 657     private void save(CdmBase cdmBase, Abcd206ImportState state) {
 658         ICdmApplicationConfiguration cdmRepository = state.getConfig().getCdmAppController();
 659         if (cdmRepository == null){
 660             cdmRepository = this;
 661         }
 662
 663         if (cdmBase.isInstanceOf(LanguageString.class)){
 664             cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 665         }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 666             cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 667         }else if (cdmBase.isInstanceOf(Reference.class)){
 668             cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 669         }else if (cdmBase.isInstanceOf(Classification.class)){
 670             cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 671         }else if (cdmBase.isInstanceOf(AgentBase.class)){
 672             cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 673         }else if (cdmBase.isInstanceOf(Collection.class)){
 674             cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 675         }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 676             cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 677         }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 678             cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 679         }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 680             cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 681         }else{
 682             throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 683         }
 684
 685     }
 686
 687     /**
 688      * setCollectionData : store the collection object into the
 689      * derivedUnitFacade
 690      *
 691      * @param state
 692      */
 693     private void setCollectionData(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
 694         // set catalogue number (unitID)
 695         derivedUnitFacade.setCatalogNumber(NB(dataHolder.unitID));
 696         derivedUnitFacade.setAccessionNumber(NB(dataHolder.accessionNumber));
 697         // derivedUnitFacade.setCollectorsNumber(NB(dataHolder.collectorsNumber));
 698
 699         /*
 700          * INSTITUTION & COLLECTION
 701          */
 702         // manage institution
 703         Institution institution = this.getInstitution(NB(dataHolder.institutionCode), state);
 704         // manage collection
 705         Collection collection = this.getCollection(institution, NB(dataHolder.collectionCode), state);
 706         // link specimen & collection
 707         derivedUnitFacade.setCollection(collection);
 708     }
 709
 710     /**
 711      * getFacade : get the DerivedUnitFacade based on the recordBasis
 712      *
 713      * @return DerivedUnitFacade
 714      */
 715     private DerivedUnitFacade getFacade() {
 716         if(DEBUG) {
 717             logger.info("getFacade()");
 718         }
 719         SpecimenOrObservationType type = null;
 720
 721         // create specimen
 722         if (NB((dataHolder.recordBasis)) != null) {
 723             if (dataHolder.recordBasis.toLowerCase().startsWith("s") || dataHolder.recordBasis.toLowerCase().indexOf("specimen")>-1) {// specimen
 724                 type = SpecimenOrObservationType.PreservedSpecimen;
 725             }
 726             if (dataHolder.recordBasis.toLowerCase().startsWith("o") ||dataHolder.recordBasis.toLowerCase().indexOf("observation")>-1 ) {
 727                 type = SpecimenOrObservationType.Observation;
 728             }
 729             if (dataHolder.recordBasis.toLowerCase().indexOf("fossil")>-1){
 730                 type = SpecimenOrObservationType.Fossil;
 731             }
 732             if (dataHolder.recordBasis.toLowerCase().indexOf("living")>-1) {
 733                 type = SpecimenOrObservationType.LivingSpecimen;
 734             }
 735             if (type == null) {
 736                 logger.info("The basis of record does not seem to be known: " + dataHolder.recordBasis);
 737                 type = SpecimenOrObservationType.DerivedUnit;
 738             }
 739             // TODO fossils?
 740         } else {
 741             logger.info("The basis of record is null");
 742             type = SpecimenOrObservationType.DerivedUnit;
 743         }
 744         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 745         return derivedUnitFacade;
 746     }
 747
 748     private void getCollectorsFromXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 749         NodeList group;
 750
 751         group = root.getChildNodes();
 752         for (int i = 0; i < group.getLength(); i++) {
 753             if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 754                 group = group.item(i).getChildNodes();
 755                 break;
 756             }
 757         }
 758         dataHolder.gatheringAgentList = new ArrayList<String>();
 759         dataHolder.gatheringTeamList = new ArrayList<String>();
 760         abcdFieldGetter.getType(root);
 761         abcdFieldGetter.getGatheringPeople(root);
 762     }
 763
 764     /**
 765      * Store the unit's properties into variables Look which unit is the
 766      * preferred one Look what kind of name it is supposed to be, for the
 767      * parsing (Botanical, Zoological)
 768      *
 769      * @param racine: the root node for a single unit
 770      */
 771     private void setUnitPropertiesXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 772         try {
 773             NodeList group;
 774
 775             group = root.getChildNodes();
 776             for (int i = 0; i < group.getLength(); i++) {
 777                 if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 778                     group = group.item(i).getChildNodes();
 779                     break;
 780                 }
 781             }
 782             dataHolder.identificationList = new ArrayList<Identification>();
 783             dataHolder.statusList = new ArrayList<SpecimenTypeDesignationStatus>();
 784             dataHolder.atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
 785             dataHolder.referenceList = new ArrayList<String[]>();
 786             dataHolder.multimediaObjects = new ArrayList<String>();
 787
 788             abcdFieldGetter.getScientificNames(group);
 789             abcdFieldGetter.getType(root);
 790
 791             if(DEBUG) {
 792                 logger.info("this.identificationList "+dataHolder.identificationList.toString());
 793             }
 794             abcdFieldGetter.getIDs(root);
 795             abcdFieldGetter.getRecordBasis(root);
 796             abcdFieldGetter.getMultimedia(root);
 797             abcdFieldGetter.getNumbers(root);
 798             abcdFieldGetter.getGeolocation(root);
 799             abcdFieldGetter.getGatheringPeople(root);
 800             abcdFieldGetter.getGatheringDate(root);
 801             abcdFieldGetter.getGatheringElevation(root);
 802             boolean referencefound = abcdFieldGetter.getReferences(root);
 803             if (!referencefound) {
 804                 String[]a = {ref.getTitleCache(),"",""};
 805                 dataHolder.referenceList.add(a);
 806             }
 807
 808         } catch (Exception e) {
 809             logger.info("Error occured while parsing XML file" + e);
 810         }
 811     }
 812
 813     /**
 814      * Look if the Institution does already exist
 815      * @param institutionCode: a string with the institutioncode
 816      * @param config : the configurator
 817      * @return the Institution (existing or new)
 818      */
 819     @SuppressWarnings("rawtypes")
 820     private Institution getInstitution(String institutionCode, Abcd206ImportState state) {
 821         Institution institution=null;
 822         List<Institution> institutions;
 823         try {
 824             institutions = getAgentService().list(Institution.class, null, null, null, null);
 825         } catch (Exception e) {
 826             institutions = new ArrayList<Institution>();
 827             logger.warn(e);
 828         }
 829         if (institutions.size() > 0 && state.getConfig().isReUseExistingMetadata()) {
 830             for (Institution institut:institutions){
 831                 try{
 832                     if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 833                         institution=institut;
 834                     }
 835                 }catch(Exception e){logger.warn("no institution code in the db");}
 836             }
 837         }
 838         if(DEBUG) {
 839             if(institution !=null) {
 840                 logger.info("getinstitution " + institution.toString());
 841             }
 842         }
 843         if (institution == null){
 844             // create institution
 845             institution = Institution.NewInstance();
 846             institution.setCode(institutionCode);
 847             institution.setTitleCache(institutionCode, true);
 848         }
 849         save(institution, state);
 850         return institution;
 851     }
 852
 853     /**
 854      * Look if the Collection does already exist
 855      * @param collectionCode
 856      * @param collectionCode: a string
 857      * @param config : the configurator
 858      * @return the Collection (existing or new)
 859      */
 860     private Collection getCollection(Institution institution, String collectionCode, Abcd206ImportState state) {
 861         Collection collection = null;
 862         List<Collection> collections;
 863         try {
 864             collections = getCollectionService().list(Collection.class, null, null, null, null);
 865         } catch (Exception e) {
 866             collections = new ArrayList<Collection>();
 867         }
 868         if (collections.size() > 0 && state.getConfig().isReUseExistingMetadata()) {
 869             for (Collection coll:collections){
 870                 if (coll.getInstitute() != null) {
 871                     if (coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 872                         collection=coll;
 873                     }
 874                 }
 875             }
 876         }
 877
 878         if(collection == null){
 879             collection =Collection.NewInstance();
 880             collection.setCode(collectionCode);
 881             collection.setInstitute(institution);
 882             collection.setTitleCache(collectionCode);
 883         }
 884         save(collection, state);
 885         return collection;
 886     }
 887
 888
 889     /**
 890      * join DeterminationEvent to the Taxon Object
 891      * @param state : the ABCD import state
 892      * @param taxon: the current Taxon
 893      * @param preferredFlag :if the current name is preferred
 894      * @param derivedFacade : the derived Unit Facade
 895      */
 896     @SuppressWarnings("rawtypes")
 897     private void linkDeterminationEvent(Abcd206ImportState state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
 898         Abcd206ImportConfigurator config = state.getConfig();
 899         if(DEBUG){
 900             logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
 901         }
 902
 903         DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 904         determinationEvent.setTaxon(taxon);
 905         determinationEvent.setPreferredFlag(preferredFlag);
 906
 907         determinationEvent.setIdentifiedUnit(derivedUnitBase);
 908         derivedUnitBase.addDetermination(determinationEvent);
 909
 910         if(DEBUG){
 911             logger.info("NB TYPES INFO: "+ dataHolder.statusList.size());
 912         }
 913         for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : dataHolder.statusList) {
 914             if (specimenTypeDesignationstatus != null) {
 915                 if(DEBUG){
 916                     logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
 917                 }
 918
 919                 ICdmApplicationConfiguration cdmAppController = config.getCdmAppController();
 920                 if(cdmAppController == null){
 921                     cdmAppController = this;
 922                 }
 923                 specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
 924                 //Designation
 925                 TaxonNameBase<?,?> name = taxon.getName();
 926                 SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
 927
 928                 designation.setTypeStatus(specimenTypeDesignationstatus);
 929                 designation.setTypeSpecimen(derivedUnitBase);
 930                 name.addTypeDesignation(designation, true);
 931             }
 932         }
 933
 934         for (String[] fullReference : dataHolder.referenceList) {
 935             List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
 936
 937             String strReference=fullReference[0];
 938             String citationDetail = fullReference[1];
 939             String citationURL = fullReference[2];
 940
 941             if (isNotBlank(strReference)){
 942                 Reference<?> reference = null;
 943                 for (Reference<?> refe: references) {
 944                     if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
 945                         reference =refe;
 946                         break;
 947                     }
 948                 }
 949                 if (reference ==null){
 950                     reference = ReferenceFactory.newGeneric();
 951                     reference.setTitleCache(strReference, true);
 952                     save(reference, state);
 953                 }
 954                 determinationEvent.addReference(reference);
 955             }
 956         }
 957         save(derivedUnitBase, state);
 958
 959         if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations()) {
 960             if(DEBUG){
 961                 logger.info("isDoCreateIndividualsAssociations");
 962             }
 963
 964             makeIndividualsAssociation(state, taxon, determinationEvent);
 965
 966             if(state.getConfig().isDeterminationOnFieldUnitLevel()){
 967                 DeterminationEvent fieldUnitDeterminationEvent = DeterminationEvent.NewInstance();
 968                 fieldUnitDeterminationEvent.setTaxon(determinationEvent.getTaxon());
 969                 fieldUnitDeterminationEvent.setPreferredFlag(determinationEvent.getPreferredFlag());
 970                 fieldUnitDeterminationEvent.setIdentifiedUnit(fieldUnit);
 971                 Set<Reference> references = determinationEvent.getReferences();
 972                 for (Reference reference : references) {
 973                     fieldUnitDeterminationEvent.addReference(reference);
 974                 }
 975                 fieldUnit.addDetermination(fieldUnitDeterminationEvent);
 976             }
 977
 978             save(derivedUnitBase, state);
 979         }
 980     }
 981
 982     /**
 983      * create and link each association (specimen, observation..) to the accepted taxon
 984      * @param state : the ABCD import state
 985      * @param taxon: the current Taxon
 986      * @param determinationEvent:the determinationevent
 987      */
 988     @SuppressWarnings("unused")
 989     private void makeIndividualsAssociation(Abcd206ImportState state, Taxon taxon, DeterminationEvent determinationEvent) {
 990         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 991
 992         if (DEBUG) {
 993             System.out.println("MAKE INDIVIDUALS ASSOCIATION");
 994         }
 995
 996         TaxonDescription taxonDescription = null;
 997         Set<TaxonDescription> descriptions= taxon.getDescriptions();
 998         if (state.getConfig().isInteractWithUser()){
 999             if(!descriptionGroupSet){
1000                 taxonDescription = sui.askForDescriptionGroup(descriptions);
1001                 descriptionGroup=taxonDescription;
1002                 descriptionGroupSet=true;
1003             }else{
1004                 taxonDescription=descriptionGroup;
1005             }
1006         } else {
1007             for (TaxonDescription description : descriptions){
1008                 Set<IdentifiableSource> sources =  description.getTaxon().getSources();
1009                 sources.addAll(description.getSources());
1010                 for (IdentifiableSource source:sources){
1011                     if(ref.equals(source.getCitation())) {
1012                         taxonDescription = description;
1013                     }
1014                 }
1015             }
1016         }
1017         if (taxonDescription == null){
1018             taxonDescription = TaxonDescription.NewInstance(taxon, false);
1019             if(sourceNotLinkedToElement(taxonDescription,ref,null)) {
1020                 taxonDescription.addSource(OriginalSourceType.Import, null, null, ref, null);
1021             }
1022             descriptionGroup=taxonDescription;
1023             taxon.addDescription(taxonDescription);
1024         }
1025
1026         //PREPARE REFERENCE QUESTIONS
1027
1028         Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1029
1030         List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1031         List<DescriptionElementSource> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1032
1033         Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1034         if(issTmp2!=null) {
1035             osbSet.addAll(issTmp2);
1036         }
1037         if(issTmp!=null) {
1038             osbSet.addAll(issTmp);
1039         }
1040
1041
1042         for( OriginalSourceBase<?> osb:osbSet) {
1043             if(osb.getCitationMicroReference() !=null && !osb.getCitationMicroReference().isEmpty()) {
1044                 try{
1045                     sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1046                 }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1047             } else{
1048                 try{
1049                     sourceMap.put(osb.getCitation().getTitleCache(),osb);
1050                 }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1051             }
1052         }
1053
1054         if (state.getConfig().isInteractWithUser()){
1055             List<OriginalSourceBase<?>> res = null;
1056             if(!descriptionSourcesSet){
1057                 res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1058                         "The current reference is "+ref.getTitleCache(),getReferenceService(), dataHolder.docSources);
1059                 descriptionRefs=res;
1060                 descriptionSourcesSet=true;
1061             }
1062             else{
1063                 res=descriptionRefs;
1064             }
1065             if(res !=null) {
1066                 for (OriginalSourceBase<?> sour:res){
1067                     if(sour.isInstanceOf(IdentifiableSource.class)){
1068                         try {
1069                             if(sourceNotLinkedToElement(taxonDescription,sour)) {
1070                                 taxonDescription.addSource((IdentifiableSource)sour.clone());
1071                             }
1072                         } catch (CloneNotSupportedException e) {
1073                             logger.warn("no cloning?");
1074                         }
1075                     }else{
1076                         if(sourceNotLinkedToElement(taxonDescription,sour)) {
1077                             taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1078                         }
1079                     }
1080                 }
1081             }
1082         }
1083         else {
1084             if(sourceNotLinkedToElement(taxonDescription,ref,null)) {
1085                 taxonDescription.addSource(OriginalSourceType.Import,null, null, ref, null);
1086             }
1087         }
1088         descriptionGroup=taxonDescription;
1089
1090         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1091         Feature feature = makeFeature(derivedUnitBase);
1092         indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
1093         indAssociation.setFeature(feature);
1094
1095         if (state.getConfig().isInteractWithUser()){
1096             sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1097
1098             issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1099             issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1100
1101             osbSet = new HashSet<OriginalSourceBase>();
1102             if(issTmp2!=null) {
1103                 osbSet.addAll(issTmp2);
1104             }
1105             if(issTmp!=null) {
1106                 osbSet.addAll(issTmp);
1107             }
1108
1109
1110             for( OriginalSourceBase<?> osb:osbSet) {
1111                 if(osb.getCitationMicroReference() !=null && !osb.getCitationMicroReference().isEmpty()) {
1112                     try{
1113                         sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1114                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1115                 } else{
1116                     try{
1117                         sourceMap.put(osb.getCitation().getTitleCache(),osb);
1118                     }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1119                 }
1120             }
1121
1122             List<OriginalSourceBase<?>> sources =null;
1123             if(!associationSourcesSet) {
1124                 sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1125                         getReferenceService(),dataHolder.docSources);
1126                 associationRefs=sources;
1127                 associationSourcesSet=true;
1128             }
1129             else{
1130                 sources=associationRefs;
1131             }
1132             if(sources !=null) {
1133                 for (OriginalSourceBase<?> source: sources) {
1134                     if(source !=null) {
1135                         if(source.isInstanceOf(DescriptionElementSource.class)){
1136                             try {
1137                                 if(sourceNotLinkedToElement(indAssociation,source)) {
1138                                     indAssociation.addSource((DescriptionElementSource)source.clone());
1139                                 }
1140                             } catch (CloneNotSupportedException e) {
1141                                 logger.warn("clone forbidden?");
1142                             }
1143                         }else{
1144                             if(sourceNotLinkedToElement(indAssociation,source)) {
1145                                 indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1146                             }
1147                             try {
1148                                 if(sourceNotLinkedToElement(derivedUnitBase, source)) {
1149                                     derivedUnitBase.addSource((IdentifiableSource) source.clone());
1150                                 }
1151                             } catch (CloneNotSupportedException e) {
1152                                 // TODO Auto-generated catch block
1153                                 e.printStackTrace();
1154                             }
1155                         }
1156
1157                     }
1158                 }
1159             }
1160         }else {
1161             if(sourceNotLinkedToElement(indAssociation,ref,null)) {
1162                 indAssociation.addSource(OriginalSourceType.Import,null, null, ref, null);
1163             }
1164             if(sourceNotLinkedToElement(derivedUnitBase, ref,null)) {
1165                 derivedUnitBase.addSource(OriginalSourceType.Import,null, null, ref, null);
1166             }
1167             for (Reference<?> citation : determinationEvent.getReferences()) {
1168                 if(sourceNotLinkedToElement(indAssociation,citation,null))
1169                 {
1170                     indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1171                 }
1172                 if(sourceNotLinkedToElement(derivedUnitBase, ref,null)) {
1173                     derivedUnitBase.addSource(OriginalSourceType.Import,null, null, ref, null);
1174                 }
1175             }
1176         }
1177
1178         taxonDescription.addElement(indAssociation);
1179
1180         save(taxonDescription, state);
1181         save(taxon, state);
1182     }
1183
1184     /**
1185      * @param derivedUnitBase2
1186      * @param ref2
1187      * @param object
1188      * @return
1189      */
1190     private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference<?> b, String d) {
1191         Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1192         for (IdentifiableSource is:linkedSources){
1193             Reference a = is.getCitation();
1194             String c = is.getCitationMicroReference();
1195
1196             boolean refMatch=false;
1197             boolean microMatch=false;
1198
1199             try{
1200                 if (a==null && b==null) {
1201                     refMatch=true;
1202                 }
1203                 if (a!=null && b!=null) {
1204                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1205                         refMatch=true;
1206                     }
1207                 }
1208             }catch(Exception e){}
1209
1210
1211             try{
1212                 if (c==null && d==null) {
1213                     microMatch=true;
1214                 }
1215                 if(c!=null && d!=null) {
1216                     if(c.equalsIgnoreCase(d)) {
1217                         microMatch=true;
1218                     }
1219                 }
1220             }
1221             catch(Exception e){}
1222
1223             if (microMatch && refMatch) {
1224                 return false;
1225             }
1226
1227
1228         }
1229         return true;
1230     }
1231
1232     /**
1233      * @param specimen
1234      * @param source
1235      * @return
1236      */
1237     private boolean sourceNotLinkedToElement(SpecimenOrObservationBase<?> specimen, OriginalSourceBase<?> source) {
1238         Set<IdentifiableSource> linkedSources = specimen.getSources();
1239         for (IdentifiableSource is:linkedSources){
1240             Reference a = is.getCitation();
1241             Reference b = source.getCitation();
1242             String c = is.getCitationMicroReference();
1243             String d = source.getCitationMicroReference();
1244
1245             boolean refMatch=false;
1246             boolean microMatch=false;
1247
1248             try{
1249                 if (a==null && b==null) {
1250                     refMatch=true;
1251                 }
1252                 if (a!=null && b!=null) {
1253                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1254                         refMatch=true;
1255                     }
1256                 }
1257             }catch(Exception e){}
1258
1259
1260             try{
1261                 if (c==null && d==null) {
1262                     microMatch=true;
1263                 }
1264                 if(c!=null && d!=null) {
1265                     if(c.equalsIgnoreCase(d)) {
1266                         microMatch=true;
1267                     }
1268                 }
1269             }
1270             catch(Exception e){}
1271
1272             if (microMatch && refMatch) {
1273                 return false;
1274             }
1275
1276
1277         }
1278         return true;
1279     }
1280
1281     /**
1282      * @param indAssociation
1283      * @param ref2
1284      * @param object
1285      * @return
1286      */
1287     private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, Reference<?> a, String d) {
1288         Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1289         for (DescriptionElementSource is:linkedSources){
1290             Reference b = is.getCitation();
1291             String c = is.getCitationMicroReference();
1292
1293             boolean refMatch=false;
1294             boolean microMatch=false;
1295
1296             try{
1297                 if (a==null && b==null) {
1298                     refMatch=true;
1299                 }
1300                 if (a!=null && b!=null) {
1301                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1302                         refMatch=true;
1303                     }
1304                 }
1305             }catch(Exception e){}
1306
1307
1308             try{
1309                 if (c==null && d==null) {
1310                     microMatch=true;
1311                 }
1312                 if(c!=null && d!=null) {
1313                     if(c.equalsIgnoreCase(d)) {
1314                         microMatch=true;
1315                     }
1316                 }
1317             }
1318             catch(Exception e){}
1319
1320             if (microMatch && refMatch) {
1321                 return false;
1322             }
1323         }
1324         return true;
1325     }
1326
1327     /**
1328      * @param taxonDescription
1329      * @param ref2
1330      * @param object
1331      * @return
1332      */
1333     private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, Reference<?> a, String d) {
1334         Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1335         for (IdentifiableSource is:linkedSources){
1336             Reference b = is.getCitation();
1337             String c = is.getCitationMicroReference();
1338
1339             boolean refMatch=false;
1340             boolean microMatch=false;
1341
1342             try{
1343                 if (a==null && b==null) {
1344                     refMatch=true;
1345                 }
1346                 if (a!=null && b!=null) {
1347                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1348                         refMatch=true;
1349                     }
1350                 }
1351             }catch(Exception e){}
1352
1353
1354             try{
1355                 if (c==null && d==null) {
1356                     microMatch=true;
1357                 }
1358                 if(c!=null && d!=null) {
1359                     if(c.equalsIgnoreCase(d)) {
1360                         microMatch=true;
1361                     }
1362                 }
1363             }
1364             catch(Exception e){}
1365
1366             if (microMatch && refMatch) {
1367                 return false;
1368             }
1369         }
1370         return true;
1371     }
1372
1373     /**
1374      * @param indAssociation
1375      * @param source
1376      * @return
1377      */
1378     private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, OriginalSourceBase<?> source) {
1379         Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1380         for (DescriptionElementSource is:linkedSources){
1381             Reference a = is.getCitation();
1382             Reference b = source.getCitation();
1383             String c = is.getCitationMicroReference();
1384             String d = source.getCitationMicroReference();
1385
1386             boolean refMatch=false;
1387             boolean microMatch=false;
1388
1389             try{
1390                 if (a==null && b==null) {
1391                     refMatch=true;
1392                 }
1393                 if (a!=null && b!=null) {
1394                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1395                         refMatch=true;
1396                     }
1397                 }
1398             }catch(Exception e){}
1399
1400
1401             try{
1402                 if (c==null && d==null) {
1403                     microMatch=true;
1404                 }
1405                 if(c!=null && d!=null) {
1406                     if(c.equalsIgnoreCase(d)) {
1407                         microMatch=true;
1408                     }
1409                 }
1410             }
1411             catch(Exception e){}
1412
1413             if (microMatch && refMatch) {
1414                 return false;
1415             }
1416         }
1417         return true;
1418     }
1419
1420     /**
1421      * @param taxonDescription
1422      * @param sour
1423      * @return
1424      */
1425     private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, OriginalSourceBase<?> sour) {
1426         Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1427         for (IdentifiableSource is:linkedSources){
1428             Reference a = is.getCitation();
1429             Reference b = sour.getCitation();
1430             String c = is.getCitationMicroReference();
1431             String d = sour.getCitationMicroReference();
1432
1433             boolean refMatch=false;
1434             boolean microMatch=false;
1435
1436             try{
1437                 if (a==null && b==null) {
1438                     refMatch=true;
1439                 }
1440                 if (a!=null && b!=null) {
1441                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1442                         refMatch=true;
1443                     }
1444                 }
1445             }catch(Exception e){}
1446
1447
1448             try{
1449                 if (c==null && d==null) {
1450                     microMatch=true;
1451                 }
1452                 if(c!=null && d!=null) {
1453                     if(c.equalsIgnoreCase(d)) {
1454                         microMatch=true;
1455                     }
1456                 }
1457             }
1458             catch(Exception e){}
1459
1460             if (microMatch && refMatch) {
1461                 return false;
1462             }
1463
1464
1465         }
1466         return true;
1467     }
1468
1469     /**
1470      * look for the Feature object (FieldObs, Specimen,...)
1471      * @param unit : a specimen or obersvation base
1472      * @return the corresponding Feature
1473      */
1474     private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1475         SpecimenOrObservationType type = unit.getRecordBasis();
1476
1477
1478
1479         if (type.isFeatureObservation()){
1480             return Feature.OBSERVATION();
1481         }else if (type.isFeatureSpecimen()){
1482             return Feature.SPECIMEN();
1483         }else if (type == SpecimenOrObservationType.DerivedUnit){
1484             return Feature.OBSERVATION();
1485             //            return getFeature("Specimen or observation");
1486         }else{
1487             String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1488             logger.warn(String.format(message, type.getMessage()));
1489             return Feature.OBSERVATION();
1490             //            return getFeature("Specimen or observation");
1491
1492         }
1493     }
1494
1495     private Feature getFeature(String featureName, Abcd206ImportState state){
1496         List<Feature> features = getTermService().list(Feature.class, null,null,null,null);
1497         Feature currentFeature=null;
1498         for (Feature feature: features){
1499             String tmpF = feature.getTitleCache();
1500             if (tmpF.equalsIgnoreCase(featureName)) {
1501                 currentFeature=feature;
1502             }
1503         }
1504         if (currentFeature == null) {
1505             currentFeature=Feature.NewInstance(featureName, featureName, featureName);
1506             save(currentFeature, state);
1507         }
1508         return currentFeature;
1509     }
1510
1511     /**
1512      *  Search for an existing taxon in the database and <b>create</b> it if it is not found.
1513      * @param state : the ABCD import state
1514      * @param scientificName : the name (string)
1515      * @param i : the current unit position in the abcd file
1516      * @param rank : the rank for the taxon
1517      * @return a Taxon
1518      */
1519     @SuppressWarnings("rawtypes")
1520     private Taxon getOrCreateTaxon(Abcd206ImportState state, String scientificName, int i, Rank rank) {
1521 //        System.out.println("GETTAXON "+scientificName);
1522         Abcd206ImportConfigurator config = state.getConfig();
1523         Taxon taxon = null;
1524         NonViralName<?> taxonName = null;
1525
1526         SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
1527
1528 //        System.out.println("config.isReuseExistingTaxaWhenPossible() :"+config.isReuseExistingTaxaWhenPossible());
1529         if (config.isReuseExistingTaxaWhenPossible()){
1530             try {
1531                 List<TaxonBase> taxonbaseList = getTaxonService().listByTitle(Taxon.class, scientificName, MatchMode.BEGINNING, null, null, null, null, null);
1532                 if (taxonbaseList.size()>0){
1533                     if(config.isInteractWithUser() && config.isAllowReuseOtherClassifications()){
1534                         taxon = sui.askWhereToFixData(scientificName,taxonbaseList, classification);
1535                     } else {
1536                         taxon = sui.lookForTaxaIntoCurrentClassification(taxonbaseList, classification);
1537                     }
1538                 }
1539                 else{
1540                     List<TaxonBase> searchByNameList = getTaxonService().searchTaxaByName(scientificName, ref);
1541                     if(config.isInteractWithUser() && config.isAllowReuseOtherClassifications()){
1542                         taxon = sui.askWhereToFixData(scientificName,searchByNameList, classification);
1543                     }
1544                     else{
1545                         taxon = sui.lookForTaxaIntoCurrentClassification(searchByNameList, classification);
1546                     }
1547                 }
1548             } catch (Exception e) {
1549                 logger.info("Searchtaxabyname failed" + e);
1550                 taxon = null;
1551             }
1552         }
1553         if (!config.isReuseExistingTaxaWhenPossible() || taxon == null){
1554 //            System.out.println("create new taxonName instance "+i+", "+config.isParseNameAutomatically());
1555             if (config.isParseNameAutomatically()){
1556                 taxonName = parseScientificName(scientificName);
1557             }
1558             else{
1559                 if (i>=0 && (dataHolder.atomisedIdentificationList != null || dataHolder.atomisedIdentificationList.size() > 0)) {
1560                     taxonName = setTaxonNameByType(dataHolder.atomisedIdentificationList.get(i), scientificName);
1561                 } else {
1562                     taxonName=null;
1563                 }
1564             }
1565             //            if (taxonName != null) {
1566             //                System.out.println(taxonName.getTitleCache());
1567             //            } else {
1568             //                System.out.println("taxonname: "+taxonName);
1569             //            }
1570             if(taxonName == null){
1571                 taxonName = NonViralName.NewInstance(rank);
1572                 taxonName.setFullTitleCache(scientificName,true);
1573                 taxonName.setTitleCache(scientificName, true);
1574             }
1575 //            System.out.println("ADD NEW TAXON *"+taxonName.getRank()+"*"+taxonName.getTitleCache());
1576             if (rank != null && (taxonName.getRank() ==null || taxonName.getRank().toString().trim().isEmpty())) {
1577                 taxonName.setRank(rank);
1578             }
1579             save(taxonName, state);
1580             taxon = Taxon.NewInstance(taxonName, ref); //sec set null
1581             save(taxon, state);
1582         }
1583         return taxon;
1584     }
1585
1586     /**
1587      * HandleIdentifications : get the scientific names present in the ABCD
1588      * document and store link them with the observation/specimen data
1589      * @param state: the current ABCD import state
1590      * @param derivedUnitFacade : the current derivedunitfacade
1591      */
1592     private void handleIdentifications(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
1593         Abcd206ImportConfigurator config = state.getConfig();
1594
1595         Taxon taxon = null;
1596
1597         String scientificName = "";
1598         boolean preferredFlag = false;
1599
1600         if (dataHolder.nomenclatureCode == ""){
1601             dataHolder.nomenclatureCode = config.getNomenclaturalCode().toString();
1602         }
1603
1604         for (int i = 0; i < dataHolder.identificationList.size(); i++) {
1605             Identification identification = dataHolder.identificationList.get(i);
1606             scientificName = identification.getScientificName().replaceAll(" et ", " & ");
1607
1608             String preferred = identification.getPreferred();
1609             if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || dataHolder.identificationList.size()==1) {
1610                 preferredFlag = true;
1611             }
1612             else {
1613                 preferredFlag = false;
1614             }
1615
1616             if (identification.getCode().indexOf(':') != -1) {
1617                 dataHolder.nomenclatureCode = identification.getCode().split(COLON)[1];
1618             }
1619             else{
1620                 dataHolder.nomenclatureCode = identification.getCode();
1621             }
1622             taxon = getOrCreateTaxon(state, scientificName, i,null);
1623             addTaxonNode(taxon, state,preferredFlag);
1624             linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
1625         }
1626     }
1627
1628
1629     private TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, Abcd206ImportState state){
1630         List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
1631         if(names.size()>0){
1632             if(names.size()>1){
1633                 logger.warn("More than one taxon name was found for "+scientificName+"!");
1634                 logger.warn("The first one found was chosen.");
1635             }
1636             return names.iterator().next();
1637         }
1638
1639         NonViralName taxonName = NonViralName.NewInstance(rank);
1640         taxonName.setFullTitleCache(scientificName,true);
1641         taxonName.setTitleCache(scientificName, true);
1642         if (rank != null) {
1643             taxonName.setRank(rank);
1644         }
1645         save(taxonName, state);
1646
1647         return taxonName;
1648     }
1649
1650     private TaxonBase<?> getTaxonForName(TaxonNameBase<?, ?> taxonNameBase){
1651         Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
1652         if(acceptedTaxa.size()>0){
1653             if(acceptedTaxa.size()>1){
1654                 logger.warn("More than one accepted taxon was found for taxon name: "+taxonNameBase.getTitleCache()+"!");
1655                 logger.warn("the first one was chosen");
1656             }
1657             return acceptedTaxa.iterator().next();
1658         }
1659         else{
1660             Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
1661             if(taxonAndSynonyms.size()>0){
1662                 if(taxonAndSynonyms.size()>1){
1663                     logger.warn("More than one synonym was found for taxon name: "+taxonNameBase.getTitleCache()+"!");
1664                     logger.warn("the first one was chosen");
1665                 }
1666                 return taxonAndSynonyms.iterator().next();
1667             }
1668         }
1669         return null;
1670     }
1671
1672     /**
1673      * @param taxon : a taxon to add as a node
1674      * @param state : the ABCD import state
1675      */
1676     private void addTaxonNode(Taxon taxon, Abcd206ImportState state, boolean preferredFlag) {
1677         logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1678         boolean exist = false;
1679         for (TaxonNode p : classification.getAllNodes()){
1680             try{
1681                 if(p.getTaxon().equals(taxon)) {
1682                     exist =true;
1683                 }
1684             }
1685             catch(Exception e){
1686                 logger.warn("TaxonNode doesn't seem to have a taxon");
1687             }
1688         }
1689         if (!exist){
1690             addParentTaxon(taxon, state, preferredFlag);
1691         }
1692     }
1693
1694     private boolean hasTaxonNodeInClassification(Taxon taxon){
1695         for (TaxonNode node : taxon.getTaxonNodes()){
1696             if(node.getClassification().equals(classification)){
1697                 return true;
1698             }
1699         }
1700         return false;
1701     }
1702
1703     /**
1704      * Add the hierarchy for a Taxon(add higher taxa)
1705      * @param taxon: a taxon to add as a node
1706      * @param state: the ABCD import state
1707      */
1708     private void addParentTaxon(Taxon taxon, Abcd206ImportState state, boolean preferredFlag){
1709         NonViralName<?>  nvname = CdmBase.deproxy(taxon.getName(), NonViralName.class);
1710         Rank rank = nvname.getRank();
1711         Taxon genus =null;
1712         Taxon subgenus =null;
1713         Taxon species = null;
1714         Taxon subspecies = null;
1715         Taxon parent = null;
1716         if (rank.isLower(Rank.GENUS() )){
1717             String prefix = nvname.getGenusOrUninomial();
1718             genus = getOrCreateTaxon(state, prefix, -1, Rank.GENUS());
1719             if (preferredFlag) {
1720                 parent = saveOrUpdateClassification(null, genus, state);
1721             }
1722
1723         }
1724         if (rank.isLower(Rank.SUBGENUS())){
1725             String prefix = nvname.getGenusOrUninomial();
1726             String name = nvname.getInfraGenericEpithet();
1727             if (name != null){
1728                 subgenus = getOrCreateTaxon(state, prefix+" "+name, -1, Rank.SUBGENUS());
1729                 if (preferredFlag) {
1730                     parent = saveOrUpdateClassification(genus, subgenus, state);
1731                 }            }
1732         }
1733         if (rank.isLower(Rank.SPECIES())){
1734             if (subgenus!=null){
1735                 String prefix = nvname.getGenusOrUninomial();
1736                 String name = nvname.getInfraGenericEpithet();
1737                 String spe = nvname.getSpecificEpithet();
1738                 if (spe != null){
1739                     species = getOrCreateTaxon(state, prefix+" "+name+" "+spe, -1, Rank.SPECIES());
1740                     if (preferredFlag) {
1741                         parent = saveOrUpdateClassification(subgenus, species, state);
1742                     }
1743                 }
1744             }
1745             else{
1746                 String prefix = nvname.getGenusOrUninomial();
1747                 String name = nvname.getSpecificEpithet();
1748                 if (name != null){
1749                     species = getOrCreateTaxon(state, prefix+" "+name, -1, Rank.SPECIES());
1750                     if (preferredFlag) {
1751                         parent = saveOrUpdateClassification(genus, species, state);
1752                     }
1753                 }
1754             }
1755         }
1756         if (rank.isLower(Rank.INFRASPECIES())){
1757             subspecies = getOrCreateTaxon(state, nvname.getFullTitleCache(), -1, Rank.SUBSPECIES());
1758             if (preferredFlag) {
1759                 parent = saveOrUpdateClassification(species, subspecies, state);
1760             }
1761         }
1762         if (preferredFlag) {
1763             saveOrUpdateClassification(parent, taxon, state);
1764         }
1765     }
1766
1767     /**
1768      * Link a parent to a child and save it in the current classification
1769      * @param parent: the higher Taxon
1770      * @param child : the lower (or current) Taxon
1771      * return the Taxon from the new created Node
1772      * @param state
1773      */
1774     private Taxon saveOrUpdateClassification(Taxon parent, Taxon child, Abcd206ImportState state) {
1775         TaxonNode node =null;
1776         if (parent != null) {
1777             parent = (Taxon) getTaxonService().find(parent.getUuid());
1778             child = (Taxon) getTaxonService().find(child.getUuid());
1779             //here we do not have to check if the taxon nodes already exists
1780             //this is done by classification.addParentChild()
1781             //do not add child node if it already exists
1782             if(hasTaxonNodeInClassification(child)){
1783                 return child;
1784             }
1785             node = classification.addParentChild(parent, child, ref, "");
1786         }
1787         else {
1788             child = (Taxon) getTaxonService().find(child.getUuid());
1789             //do not add child node if it already exists
1790             if(hasTaxonNodeInClassification(child)){
1791                 return child;
1792             }
1793             node =classification.addChildTaxon(child, ref, null);
1794         }
1795         save(classification, state);
1796         return node.getTaxon();
1797     }
1798
1799     /**
1800      * Parse automatically the scientific name
1801      * @param scientificName: the scientific name to parse
1802      * @return a parsed name
1803      */
1804     private NonViralName<?> parseScientificName(String scientificName) {
1805         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1806         NonViralName<?> taxonName = null;
1807         boolean problem = false;
1808
1809         if(DEBUG){
1810             logger.info("parseScientificName " + dataHolder.nomenclatureCode.toString());
1811         }
1812
1813         if (dataHolder.nomenclatureCode.toString().equals("Zoological") || dataHolder.nomenclatureCode.toString().contains("ICZN")) {
1814             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
1815             if (taxonName.hasProblem()) {
1816                 problem = true;
1817             }
1818         }
1819         if (dataHolder.nomenclatureCode.toString().equals("Botanical") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1820             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
1821             if (taxonName.hasProblem()) {
1822                 problem = true;
1823             }
1824         }
1825         if (dataHolder.nomenclatureCode.toString().equals("Bacterial") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1826             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
1827             if (taxonName.hasProblem()) {
1828                 problem = true;
1829             }
1830         }
1831         if (dataHolder.nomenclatureCode.toString().equals("Cultivar") || dataHolder.nomenclatureCode.toString().contains("ICNCP")) {
1832             taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
1833             if (taxonName.hasProblem()) {
1834                 problem = true;
1835             }
1836         }
1837         if (problem) {
1838             logger.info("Parsing with problem in parseScientificName " + scientificName);
1839             return null;
1840         }
1841         return taxonName;
1842
1843     }
1844
1845     /**
1846      * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
1847      * The name is built upon the ABCD fields
1848      * @param atomisedMap : the ABCD atomised fields
1849      * @param fullName : the full scientific name
1850      * @return the corresponding Botanical or Zoological or... name
1851      */
1852     private NonViralName<?> setTaxonNameByType(
1853             HashMap<String, String> atomisedMap, String fullName) {
1854         boolean problem = false;
1855         if(DEBUG) {
1856             logger.info("settaxonnamebytype " + dataHolder.nomenclatureCode.toString());
1857         }
1858
1859         if (dataHolder.nomenclatureCode.equals("Zoological")) {
1860             NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
1861             taxonName.setFullTitleCache(fullName, true);
1862             taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1863             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1864             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
1865             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
1866
1867             if (taxonName.getGenusOrUninomial() != null){
1868                 taxonName.setRank(Rank.GENUS());
1869             }
1870
1871             if (taxonName.getInfraGenericEpithet() != null){
1872                 taxonName.setRank(Rank.SUBGENUS());
1873             }
1874
1875             if (taxonName.getSpecificEpithet() != null){
1876                 taxonName.setRank(Rank.SPECIES());
1877             }
1878
1879             if (taxonName.getInfraSpecificEpithet() != null){
1880                 taxonName.setRank(Rank.SUBSPECIES());
1881             }
1882
1883             Team team = null;
1884             if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1885                 team = Team.NewInstance();
1886                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1887             }
1888             else {
1889                 if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1890                     team = Team.NewInstance();
1891                     team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1892                 }
1893             }
1894             if (team != null) {
1895                 taxonName.setBasionymAuthorTeam(team);
1896             }
1897             else {
1898                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1899                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1900                 }
1901                 else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1902                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
1903                 }
1904             }
1905             if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1906                 team = Team.NewInstance();
1907                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1908                 taxonName.setCombinationAuthorTeam(team);
1909             }
1910             if (taxonName.hasProblem()) {
1911                 logger.info("pb ICZN");
1912                 problem = true;
1913             }
1914             else {
1915                 return taxonName;
1916             }
1917         }
1918         else if (dataHolder.nomenclatureCode.equals("Botanical")) {
1919             BotanicalName taxonName = (BotanicalName) parseScientificName(fullName);
1920             if (taxonName != null){
1921                 return taxonName;
1922             }
1923             else{
1924                 taxonName = BotanicalName.NewInstance(null);
1925             }
1926             taxonName.setFullTitleCache(fullName, true);
1927             taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1928             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
1929             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
1930             try {
1931                 taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
1932             } catch (Exception e) {
1933                 if (taxonName.getGenusOrUninomial() != null){
1934                     taxonName.setRank(Rank.GENUS());
1935                 }
1936                 else if (taxonName.getInfraGenericEpithet() != null){
1937                     taxonName.setRank(Rank.SUBGENUS());
1938                 }
1939                 else if (taxonName.getSpecificEpithet() != null){
1940                     taxonName.setRank(Rank.SPECIES());
1941                 }
1942                 else if (taxonName.getInfraSpecificEpithet() != null){
1943                     taxonName.setRank(Rank.SUBSPECIES());
1944                 }
1945             }
1946             Team team = null;
1947             if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1948                 team = Team.NewInstance();
1949                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1950                 taxonName.setBasionymAuthorTeam(team);
1951             }
1952             if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1953                 team = Team.NewInstance();
1954                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
1955                 taxonName.setCombinationAuthorTeam(team);
1956             }
1957             if (team == null) {
1958                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1959                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1960                 }
1961                 else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1962                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
1963                 }
1964             }
1965             if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1966                 team = Team.NewInstance();
1967                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1968                 taxonName.setCombinationAuthorTeam(team);
1969             }
1970             if (taxonName.hasProblem()) {
1971                 logger.info("pb ICBN");
1972                 problem = true;
1973             }
1974             else {
1975                 return taxonName;
1976             }
1977         }
1978         else if (dataHolder.nomenclatureCode.equals("Bacterial")) {
1979             NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
1980             taxonName.setFullTitleCache(fullName, true);
1981             taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
1982             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1983             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
1984             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
1985
1986             if (taxonName.getGenusOrUninomial() != null){
1987                 taxonName.setRank(Rank.GENUS());
1988             }
1989             else if (taxonName.getInfraGenericEpithet() != null){
1990                 taxonName.setRank(Rank.SUBGENUS());
1991             }
1992             else if (taxonName.getSpecificEpithet() != null){
1993                 taxonName.setRank(Rank.SPECIES());
1994             }
1995             else if (taxonName.getInfraSpecificEpithet() != null){
1996                 taxonName.setRank(Rank.SUBSPECIES());
1997             }
1998
1999             if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
2000                 Team team = Team.NewInstance();
2001                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
2002                 taxonName.setCombinationAuthorTeam(team);
2003             }
2004             if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
2005                 Team team = Team.NewInstance();
2006                 team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
2007                 taxonName.setBasionymAuthorTeam(team);
2008             }
2009             if (taxonName.hasProblem()) {
2010                 logger.info("pb ICNB");
2011                 problem = true;
2012             }
2013             else {
2014                 return taxonName;
2015             }
2016         }
2017         else if (dataHolder.nomenclatureCode.equals("Cultivar")) {
2018             CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
2019
2020             if (taxonName.hasProblem()) {
2021                 logger.info("pb ICNCP");
2022                 problem = true;
2023             }
2024             else {
2025                 return taxonName;
2026             }
2027             return taxonName;
2028         }
2029
2030         if (problem) {
2031             logger.info("Problem im setTaxonNameByType ");
2032             NonViralName<?> taxonName = NonViralName.NewInstance(null);
2033             taxonName.setFullTitleCache(fullName, true);
2034             return taxonName;
2035         }
2036         NonViralName<?> tn = NonViralName.NewInstance(null);
2037         return tn;
2038     }
2039
2040
2041     /**
2042      * Get a formated string from a hashmap
2043      * @param atomisedMap
2044      * @param key
2045      * @return
2046      */
2047     private String getFromMap(HashMap<String, String> atomisedMap, String key) {
2048         String value = null;
2049         if (atomisedMap.containsKey(key)) {
2050             value = atomisedMap.get(key);
2051         }
2052
2053         try {
2054             if (value != null && key.matches(".*Year.*")) {
2055                 value = value.trim();
2056                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
2057                     String tmp = value.split("[0-9]{4}$")[0];
2058                     int year = Integer.parseInt(value.split(tmp)[1]);
2059                     if (year >= 1752) {
2060                         value = tmp;
2061                     }
2062                     else {
2063                         value = null;
2064                     }
2065                 }
2066                 else {
2067                     value = null;
2068                 }
2069             }
2070         }
2071         catch (Exception e) {
2072             value = null;
2073         }
2074         return value;
2075     }
2076
2077     //    private void compareABCDtoCDM(URI urlFileName, List<String> knownElts, Abcd206XMLFieldGetter abcdFieldGetter) {
2078     //        try {
2079     //            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
2080     //            DocumentBuilder constructeur = factory.newDocumentBuilder();
2081     //            URL url = urlFileName.toURL();
2082     //            Object o = url.getContent();
2083     //            InputStream is = (InputStream) o;
2084     //            Document document = constructeur.parse(is);
2085     //            Element root = document.getDocumentElement();
2086     //            abcdFieldGetter.traverse(root);
2087     //        }
2088     //        catch (ParserConfigurationException e){
2089     //            e.printStackTrace();
2090     //        }
2091     //        catch (SAXException e) {
2092     //            e.printStackTrace();
2093     //        }
2094     //        catch (IOException e) {
2095     //            e.printStackTrace();
2096     //        }
2097     //        Set<String> elts = dataHolder.allABCDelements.keySet();
2098     //        Iterator<String> it = elts.iterator();
2099     //        String elt;
2100     //        while (it.hasNext()) {
2101     //            elt = it.next();
2102     //            if (knownElts.indexOf(elt) == -1) {
2103     //                if(DEBUG) {
2104     //                    logger.info("Unmerged ABCD element: " + elt + " - "+ dataHolder.allABCDelements.get(elt));
2105     //                }
2106     //            }
2107     //        }
2108     //    }
2109
2110     /**
2111      * Load the list of names from the ABCD file and save them
2112      * @param state : the current ABCD import state
2113      * @param unitsList : the unit list from the ABCD file
2114      * @param abcdFieldGetter : the ABCD parser
2115      */
2116     private void prepareCollectors(Abcd206ImportState state, NodeList unitsList, Abcd206XMLFieldGetter abcdFieldGetter) {
2117         List<String> collectors = new ArrayList<String>();
2118         List<String> teams = new ArrayList<String>();
2119         List<List<String>> collectorinteams = new ArrayList<List<String>>();
2120
2121         for (int i = 0; i < unitsList.getLength(); i++) {
2122             this.getCollectorsFromXML((Element) unitsList.item(i), abcdFieldGetter);
2123             for (String agent : dataHolder.gatheringAgentList) {
2124                 collectors.add(agent);
2125             }
2126             List<String> tmpTeam = new ArrayList<String>(new HashSet<String>(dataHolder.gatheringTeamList));
2127             if(!tmpTeam.isEmpty()) {
2128                 teams.add(StringUtils.join(tmpTeam.toArray()," & "));
2129             }
2130             for (String agent:tmpTeam) {
2131                 collectors.add(agent);
2132             }
2133         }
2134
2135         List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
2136         List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
2137
2138
2139         //existing teams in DB
2140         Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
2141         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
2142
2143         Set<UUID> uuids = new HashSet<UUID>();
2144         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
2145             uuids.add(hibernateT.getUuid());
2146         }
2147         if (!uuids.isEmpty()){
2148             List<AgentBase> existingTeams = getAgentService().find(uuids);
2149             for (AgentBase<?> existingP:existingTeams){
2150                 titleCacheTeam.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Team.class));
2151             }
2152         }
2153
2154
2155         Map<String,UUID> teamMap = new HashMap<String, UUID>();
2156         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
2157             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
2158         }
2159
2160         //existing persons in DB
2161         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
2162         Map<String,Person> titleCachePerson = new HashMap<String, Person>();
2163         uuids = new HashSet<UUID>();
2164         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
2165             uuids.add(hibernateP.getUuid());
2166         }
2167
2168         if (!uuids.isEmpty()){
2169             List<AgentBase> existingPersons = getAgentService().find(uuids);
2170             for (AgentBase<?> existingP:existingPersons){
2171                 titleCachePerson.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Person.class));
2172             }
2173         }
2174
2175         Map<String,UUID> personMap = new HashMap<String, UUID>();
2176         for (UuidAndTitleCache<Person> person:hiberPersons){
2177             personMap.put(person.getTitleCache(), person.getUuid());
2178         }
2179
2180         java.util.Collection<Person> personToadd = new ArrayList<Person>();
2181         java.util.Collection<Team> teamToAdd = new ArrayList<Team>();
2182
2183         for (String collector:collectorsU){
2184             Person p = Person.NewInstance();
2185             p.setTitleCache(collector,true);
2186             if (!personMap.containsKey(p.getTitleCache())){
2187                 personToadd.add(p);
2188             }
2189         }
2190         for (String team:teamsU){
2191             Team p = Team.NewInstance();
2192             p.setTitleCache(team,true);
2193             if (!teamMap.containsKey(p.getTitleCache())){
2194                 teamToAdd.add(p);
2195             }
2196         }
2197
2198         if(!personToadd.isEmpty()){
2199             for (Person agent: personToadd){
2200                 save(agent, state);
2201                 titleCachePerson.put(agent.getTitleCache(),CdmBase.deproxy(agent, Person.class) );
2202             }
2203         }
2204
2205         Person ptmp ;
2206         Map <String,Integer>teamdone = new HashMap<String, Integer>();
2207         for (List<String> collteam: collectorinteams){
2208             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
2209                 Team team = new Team();
2210                 boolean em =true;
2211                 for (String collector:collteam){
2212                     ptmp = Person.NewInstance();
2213                     ptmp.setTitleCache(collector,true);
2214                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
2215                     team.addTeamMember(p2);
2216                     em=false;
2217                 }
2218                 if (!em) {
2219                     teamToAdd.add(team);
2220                 }
2221                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
2222             }
2223         }
2224
2225         if(!teamToAdd.isEmpty()){
2226             for (Team agent: teamToAdd){
2227                 save(agent, state);
2228                 titleCacheTeam.put(agent.getTitleCache(), CdmBase.deproxy( agent,Team.class) );
2229             }
2230         }
2231
2232         state.getConfig().setTeams(titleCacheTeam);
2233         state.getConfig().setPersons(titleCachePerson);
2234     }
2235
2236     @Override
2237     protected boolean isIgnore(Abcd206ImportState state) {
2238         return false;
2239     }
2240
2241 }