cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.net.MalformedURLException;
  15 import java.net.URI;
  16 import java.net.URL;
  17 import java.util.ArrayList;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24 import java.util.UUID;
  25
  26 import javax.xml.parsers.DocumentBuilder;
  27 import javax.xml.parsers.DocumentBuilderFactory;
  28 import javax.xml.parsers.ParserConfigurationException;
  29
  30 import org.apache.commons.lang.StringUtils;
  31 import org.apache.log4j.Logger;
  32 import org.springframework.stereotype.Component;
  33 import org.w3c.dom.Document;
  34 import org.w3c.dom.Element;
  35 import org.w3c.dom.NodeList;
  36 import org.xml.sax.SAXException;
  37
  38 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  39 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  40 import eu.etaxonomy.cdm.io.specimen.SpecimenImportBase;
  41 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  42 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  43 import eu.etaxonomy.cdm.model.agent.AgentBase;
  44 import eu.etaxonomy.cdm.model.agent.Institution;
  45 import eu.etaxonomy.cdm.model.agent.Person;
  46 import eu.etaxonomy.cdm.model.agent.Team;
  47 import eu.etaxonomy.cdm.model.common.CdmBase;
  48 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  49 import eu.etaxonomy.cdm.model.common.Language;
  50 import eu.etaxonomy.cdm.model.common.UuidAndTitleCache;
  51 import eu.etaxonomy.cdm.model.description.Feature;
  52 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  53 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  54 import eu.etaxonomy.cdm.model.location.NamedArea;
  55 import eu.etaxonomy.cdm.model.media.Media;
  56 import eu.etaxonomy.cdm.model.name.BacterialName;
  57 import eu.etaxonomy.cdm.model.name.BotanicalName;
  58 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  59 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  60 import eu.etaxonomy.cdm.model.name.NonViralName;
  61 import eu.etaxonomy.cdm.model.name.Rank;
  62 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  63 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  64 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  65 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  66 import eu.etaxonomy.cdm.model.occurrence.Collection;
  67 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  68 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
  69 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  70 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
  71 import eu.etaxonomy.cdm.model.occurrence.Fossil;
  72 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  73 import eu.etaxonomy.cdm.model.occurrence.LivingBeing;
  74 import eu.etaxonomy.cdm.model.occurrence.Observation;
  75 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  76 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  77 import eu.etaxonomy.cdm.model.reference.Reference;
  78 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  79 import eu.etaxonomy.cdm.model.taxon.Classification;
  80 import eu.etaxonomy.cdm.model.taxon.Taxon;
  81 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  82 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  83 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  84
  85 /**
  86  * @author p.kelbert
  87  * @created 20.10.2008
  88  * @version 1.0
  89  */
  90 @Component
  91 public class Abcd206Import extends SpecimenImportBase<Abcd206ImportConfigurator, Abcd206ImportState> {
  92         private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  93
  94
  95         private final boolean DEBUG = false;
  96
  97         private static final String SEC = "sec. ";
  98         private static final String PREFERRED = "_preferred_";
  99         private static final String CODE = "_code_";
 100         private static final String COLON = ":";
 101         private static final String SPLITTER = "--";
 102
 103
 104         private static String prefix = "";
 105
 106         //TODO make all fields ABCD206ImportState variables
 107         private Classification classification = null;
 108         private Reference<?> ref = null;
 109
 110         private Abcd206DataHolder dataHolder;
 111         private DerivedUnitBase<?> derivedUnitBase;
 112
 113         public Abcd206Import() {
 114                 super();
 115         }
 116
 117         @Override
 118         protected boolean doCheck(Abcd206ImportState state) {
 119                 logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
 120                 return true;
 121         }
 122
 123         /**
 124          * getClassification : get the classification declared in the ImportState
 125          *
 126          * @param state
 127          * @return
 128          */
 129         private void setClassification(Abcd206ImportState state) {
 130                 if (classification == null) {
 131                         String name = NB(state.getConfig().getClassificationName());
 132
 133                         classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 134                         if (state.getConfig().getClassificationUuid() != null) {
 135                                 classification.setUuid(state.getConfig().getClassificationUuid());
 136                         }
 137                         getClassificationService().saveOrUpdate(classification);
 138                         refreshTransaction(state);
 139                 }
 140         }
 141
 142
 143         @Override
 144         public void doInvoke(Abcd206ImportState state) {
 145                 state.setTx(startTransaction());
 146
 147                 logger.info("INVOKE Specimen Import from ABCD2.06 XML ");
 148                 URI sourceName = state.getConfig().getSource();
 149                 NodeList unitsList = getUnitsNodeList(sourceName);
 150
 151                 ref = state.getConfig().getSourceReference();
 152                 setClassification(state);
 153
 154                 if (unitsList != null) {
 155                         String message = "nb units to insert: " + unitsList.getLength();
 156                         logger.info(message);
 157                         updateProgress(state, message);
 158                         dataHolder = new Abcd206DataHolder();
 159
 160                         Abcd206XMLFieldGetter abcdFieldGetter = new Abcd206XMLFieldGetter(dataHolder, prefix);
 161
 162                         prepareCollectors(state, unitsList, abcdFieldGetter);
 163
 164                         for (int i = 0; i < unitsList.getLength(); i++) {
 165
 166                                 this.setUnitPropertiesXML( (Element) unitsList.item(i), abcdFieldGetter);
 167 //                              refreshTransaction(state);
 168                                 this.handleSingleUnit(state, i);
 169
 170                                 // compare the ABCD elements added in to the CDM and the
 171                                 // unhandled ABCD elements
 172                                 //compareABCDtoCDM(sourceName, dataHolder.knownABCDelements, abcdFieldGetter);
 173
 174                                 // reset the ABCD elements added in CDM
 175                                 // knownABCDelements = new ArrayList<String>();
 176                                 dataHolder.allABCDelements = new HashMap<String, String>();
 177
 178                                 //refreshTransaction(state);
 179                         }
 180                 }
 181                 commitTransaction(state.getTx());
 182                 return;
 183
 184         }
 185
 186         /*
 187          * Return the list of root nodes for an ABCD 2.06 XML file
 188          *
 189          * @param fileName: the file's location
 190          *
 191          * @return the list of root nodes ("Unit")
 192          */
 193         protected NodeList getUnitsNodeList(URI urlFileName) {
 194                 NodeList unitList = null;
 195                 try {
 196                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 197                         DocumentBuilder builder = factory.newDocumentBuilder();
 198                         URL url = urlFileName.toURL();
 199                         Object o = url.getContent();
 200                         InputStream is = (InputStream) o;
 201                         Document document = builder.parse(is);
 202                         Element root = document.getDocumentElement();
 203                         unitList = root.getElementsByTagName("Unit");
 204                         if (unitList.getLength() == 0) {
 205                                 unitList = root.getElementsByTagName("abcd:Unit");
 206                                 prefix = "abcd:";
 207                         }
 208                 } catch (Exception e) {
 209                         logger.warn(e);
 210                 }
 211                 return unitList;
 212         }
 213
 214         /*
 215          * Stores the unit with its Gathering informations in the CDM
 216          */
 217         private void handleSingleUnit(Abcd206ImportState state, int i) {
 218                 logger.info("handleSingleUnit");
 219
 220                 try {
 221                         updateProgress(state, "Importing data for unit: " + dataHolder.unitID);
 222
 223
 224                         // create facade
 225                         DerivedUnitFacade derivedUnitFacade = getFacade();
 226                         derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 227
 228                         /**
 229                          * GATHERING EVENT
 230                          */
 231
 232                         // gathering event
 233             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(getTermService(), dataHolder.locality, dataHolder.languageIso,
 234                     dataHolder.longitude, dataHolder.latitude, dataHolder.gatheringAgentList, dataHolder.gatheringTeamList, state.getConfig());
 235
 236                         // country
 237                         UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea(NB(dataHolder.isocountry), NB(dataHolder.country), this);
 238                         NamedArea areaCountry = unitsGatheringArea.getArea();
 239
 240                         // other areas
 241                         unitsGatheringArea = new UnitsGatheringArea(dataHolder.namedAreaList);
 242                         ArrayList<NamedArea> otherAreas = unitsGatheringArea.getAreas();
 243                         for (NamedArea namedArea : otherAreas) {
 244                                 unitsGatheringEvent.addArea(namedArea);
 245                         }
 246
 247                         // copy gathering event to facade
 248                         GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 249                         derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 250                         derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 251                         derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 252                         derivedUnitFacade.setCountry(areaCountry);
 253                         derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 254
 255                         // TODO exsiccatum
 256
 257                         // add fieldNumber
 258                         derivedUnitFacade.setFieldNumber(NB(dataHolder.fieldNumber));
 259
 260                         // //add Multimedia URLs
 261                         if (dataHolder.multimediaObjects.size() != -1) {
 262                                 for (String multimediaObject : dataHolder.multimediaObjects) {
 263                                         Media media;
 264                                         try {
 265                                                 media = getImageMedia(multimediaObject, READ_MEDIA_DATA, false);
 266                                                 derivedUnitFacade.addDerivedUnitMedia(media);
 267                                         } catch (MalformedURLException e) {
 268                                                 // TODO Auto-generated catch block
 269                                                 e.printStackTrace();
 270                                         }
 271
 272                                 }
 273                         }
 274
 275                         /*
 276                          * merge AND STORE DATA
 277                          */
 278                         getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 279
 280                         for (NamedArea area : otherAreas) {
 281                                 getTermService().saveOrUpdate(area);// merge it sooner (foreach area)
 282                         }
 283                         getTermService().saveLanguageData(unitsGatheringEvent.getLocality());
 284
 285                         // handle collection data
 286                         setCollectionData(state.getConfig(), derivedUnitFacade);
 287
 288
 289                         getOccurrenceService().saveOrUpdate(derivedUnitBase);
 290                         refreshTransaction(state);
 291
 292                         // handle identifications
 293                         handleIdentifications(state, derivedUnitFacade);
 294
 295                         logger.info("saved ABCD specimen ...");
 296
 297                 } catch (Exception e) {
 298                         logger.warn("Error when reading record!!");
 299                         e.printStackTrace();
 300                         state.setUnsuccessfull();
 301                 }
 302
 303                 return;
 304         }
 305
 306         /**
 307          * setCollectionData : store the collection object into the
 308          * derivedUnitFacade
 309          *
 310          * @param config
 311          */
 312         private void setCollectionData(Abcd206ImportConfigurator config,
 313                         DerivedUnitFacade derivedUnitFacade) {
 314                 // set catalogue number (unitID)
 315                 derivedUnitFacade.setCatalogNumber(NB(dataHolder.unitID));
 316                 derivedUnitFacade.setAccessionNumber(NB(dataHolder.accessionNumber));
 317                 // derivedUnitFacade.setCollectorsNumber(NB(dataHolder.collectorsNumber));
 318
 319                 /*
 320                  * INSTITUTION & COLLECTION
 321                  */
 322                 // manage institution
 323                 Institution institution = this.getInstitution(NB(dataHolder.institutionCode), config);
 324                 // manage collection
 325                 Collection collection = this.getCollection(institution, config);
 326                 // link specimen & collection
 327                 derivedUnitFacade.setCollection(collection);
 328         }
 329
 330         /**
 331          * getFacade : get the DerivedUnitFacade based on the recordBasis
 332          *
 333          * @return DerivedUnitFacade
 334          */
 335         private DerivedUnitFacade getFacade() {
 336         if(DEBUG) {
 337             logger.info("getFacade()");
 338         }
 339                 DerivedUnitType type = null;
 340
 341                 // create specimen
 342                 if (NB((dataHolder.recordBasis)) != null) {
 343                         if (dataHolder.recordBasis.toLowerCase().startsWith("s") || dataHolder.recordBasis.toLowerCase().contains("specimen")) {// specimen
 344                                 type = DerivedUnitType.Specimen;
 345                         }
 346                         if (dataHolder.recordBasis.toLowerCase().startsWith("o")) {
 347                                 type = DerivedUnitType.Observation;
 348                         }
 349                         if (dataHolder.recordBasis.toLowerCase().contains("fossil")){
 350                                 type = DerivedUnitType.Fossil;
 351                         }
 352                         if (dataHolder.recordBasis.toLowerCase().startsWith("l")) {
 353                                 type = DerivedUnitType.LivingBeing;
 354                         }
 355                         if (type == null) {
 356                                 logger.info("The basis of record does not seem to be known: " + dataHolder.recordBasis);
 357                                 type = DerivedUnitType.DerivedUnit;
 358                         }
 359                         // TODO fossils?
 360                 } else {
 361                         logger.info("The basis of record is null");
 362                         type = DerivedUnitType.DerivedUnit;
 363                 }
 364                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 365                 return derivedUnitFacade;
 366         }
 367
 368     private void getCollectorsFromXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 369         NodeList group;
 370
 371         group = root.getChildNodes();
 372         for (int i = 0; i < group.getLength(); i++) {
 373             if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 374                 group = group.item(i).getChildNodes();
 375                 break;
 376             }
 377         }
 378         dataHolder.gatheringAgentList = new ArrayList<String>();
 379         dataHolder.gatheringTeamList = new ArrayList<String>();
 380         abcdFieldGetter.getType(root);
 381         abcdFieldGetter.getGatheringPeople(root);
 382     }
 383
 384         /**
 385          * Store the unit's properties into variables Look which unit is the
 386          * preferred one Look what kind of name it is supposed to be, for the
 387          * parsing (Botanical, Zoological)
 388          *
 389          * @param racine: the root node for a single unit
 390          */
 391         private void setUnitPropertiesXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 392                 try {
 393                         NodeList group;
 394
 395                         group = root.getChildNodes();
 396                         for (int i = 0; i < group.getLength(); i++) {
 397                                 if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 398                                         group = group.item(i).getChildNodes();
 399                                         break;
 400                                 }
 401                         }
 402                         dataHolder.identificationList = new ArrayList<String>();
 403                         dataHolder.statusList = new ArrayList<SpecimenTypeDesignationStatus>();
 404                         dataHolder.atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
 405                         dataHolder.referenceList = new ArrayList<String>();
 406                         dataHolder.multimediaObjects = new ArrayList<String>();
 407
 408                         abcdFieldGetter.getScientificNames(group);
 409                         abcdFieldGetter.getType(root);
 410
 411             if(DEBUG) {
 412                 logger.info("this.identificationList "+dataHolder.identificationList.toString());
 413             }
 414             abcdFieldGetter.getIDs(root);
 415                         abcdFieldGetter.getRecordBasis(root);
 416                         abcdFieldGetter.getMultimedia(root);
 417                         abcdFieldGetter.getNumbers(root);
 418                         abcdFieldGetter.getGeolocation(root);
 419                         abcdFieldGetter.getGatheringPeople(root);
 420                         boolean referencefound = abcdFieldGetter.getReferences(root);
 421                         if (!referencefound) {
 422                                 dataHolder.referenceList.add(ref.getTitleCache());
 423                         }
 424
 425                 } catch (Exception e) {
 426                         logger.info("Error occured while parsing XML file" + e);
 427                 }
 428         }
 429
 430         private Institution getInstitution(String institutionCode, Abcd206ImportConfigurator config) {
 431                 Institution institution;
 432                 List<Institution> institutions;
 433                 try {
 434                         if(DEBUG){ logger.info(dataHolder.institutionCode);}
 435                         institutions = getAgentService().searchInstitutionByCode(dataHolder.institutionCode);
 436                 } catch (Exception e) {
 437                         institutions = new ArrayList<Institution>();
 438                 }
 439                 if (institutions.size() == 0 || !config.isReUseExistingMetadata()) {
 440                         if(DEBUG) {
 441                                 logger.info("Institution (agent) unknown or not allowed to reuse existing metadata");
 442                         }
 443                         // create institution
 444                         institution = Institution.NewInstance();
 445                         institution.setCode(NB(dataHolder.institutionCode));
 446                 }
 447                 else {
 448                         if(DEBUG){
 449                                 logger.info("Institution (agent) already in the db");
 450                         }
 451                         institution = institutions.get(0);
 452                 }
 453                 if(DEBUG) logger.info("getinstitution " + institution.toString());
 454                 return institution;
 455         }
 456
 457         /**
 458          * Look if the Collection does already exists
 459          * @param collectionCode: a string
 460          * @param institution: the current Institution
 461          * @param app
 462          * @return the Collection (existing or new)
 463          */
 464         private Collection getCollection(Institution institution, Abcd206ImportConfigurator config) {
 465                 Collection collection = Collection.NewInstance();
 466                 List<Collection> collections;
 467                 try {
 468                         collections = getCollectionService().searchByCode(dataHolder.collectionCode);
 469                 } catch (Exception e) {
 470                         collections = new ArrayList<Collection>();
 471                 }
 472                 if (collections.size() == 0 || !config.isReUseExistingMetadata()) {
 473                         if(DEBUG) logger.info("Collection not found or do not reuse existing metadata  " + dataHolder.collectionCode);
 474                         // create new collection
 475                         collection.setCode(NB(dataHolder.collectionCode));
 476                         collection.setCodeStandard(NB("GBIF"));
 477                         collection.setInstitute(institution);
 478                 } else {
 479                         boolean collectionFound = false;
 480                         for (int i = 0; i < collections.size(); i++) {
 481                                 collection = collections.get(i);
 482                                 try {
 483                                         if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())) {
 484                                                 // found a collection with the same code and the same institution
 485                                                 collectionFound = true;
 486                                                 break;
 487                                         }
 488                                 } catch (NullPointerException e) {
 489                                         //TODO: exception?
 490                                 }
 491                         }
 492                         if (!collectionFound) {
 493                                 collection.setCode(NB(dataHolder.collectionCode));
 494                                 collection.setCodeStandard(NB("GBIF"));
 495                                 collection.setInstitute(institution);
 496                         }
 497
 498                 }
 499                 return collection;
 500         }
 501
 502         /**
 503          * join DeterminationEvent to the Taxon Object
 504          * @param taxon current Taxon Object
 505          * @param preferredFlag preferred name, boolean
 506          * @param config current ABCD Import configurator
 507          */
 508
 509         private void linkDeterminationEvent(Abcd206ImportState state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
 510                 Abcd206ImportConfigurator config = state.getConfig();
 511
 512                 if(DEBUG){
 513                         logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
 514                 }
 515
 516 //              refreshTransaction(state);
 517
 518                 if (taxon != null){
 519                         taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 520                 }
 521
 522
 523                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 524                 determinationEvent.setTaxon(taxon);
 525                 determinationEvent.setPreferredFlag(preferredFlag);
 526
 527                 determinationEvent.setIdentifiedUnit(derivedUnitBase);
 528
 529                 derivedUnitBase.addDetermination(determinationEvent);
 530 //              refreshTransaction(state);
 531
 532                 try {
 533                         if(DEBUG){
 534                                 logger.info("NB TYPES INFO: "+ dataHolder.statusList.size());
 535                         }
 536                         for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : dataHolder.statusList) {
 537                                 if (specimenTypeDesignationstatus != null) {
 538                                         if(DEBUG){
 539                                                 logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
 540                                         }
 541                                         if (taxon != null){
 542                                                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 543                                         }
 544                                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) getTermService().find(specimenTypeDesignationstatus.getUuid());
 545                                         //Designation
 546                                         TaxonNameBase<?,?> name = taxon.getName();
 547                                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
 548
 549                                         designation.setTypeStatus(specimenTypeDesignationstatus);
 550                                         designation.setTypeSpecimen(derivedUnitBase);
 551                                         name.addTypeDesignation(designation, true);
 552                                         refreshTransaction(state);
 553
 554                                 }
 555                         }
 556                 } catch (Exception e) {
 557                         logger.warn("PB addding SpecimenType " + e);
 558                 }
 559
 560                 for (String strReference : dataHolder.referenceList) {
 561                         if (isNotBlank(strReference)){
 562                                 Reference<?> reference = ReferenceFactory.newGeneric();
 563                                 reference.setTitleCache(strReference, true);
 564                                 getReferenceService().saveOrUpdate(reference);
 565
 566                                 determinationEvent.addReference(reference);
 567                         }
 568                 }
 569
 570                 getOccurrenceService().saveOrUpdate(derivedUnitBase);
 571                 refreshTransaction(state);
 572
 573
 574                 if (config.isDoCreateIndividualsAssociations()) {
 575                         if(DEBUG){ logger.info("isDoCreateIndividualsAssociations");}
 576
 577                    makeIndividualsAssociation(state, taxon, determinationEvent);
 578                    getOccurrenceService().saveOrUpdate(derivedUnitBase);
 579                 }
 580         }
 581
 582         private void makeIndividualsAssociation(Abcd206ImportState state, Taxon taxon, DeterminationEvent determinationEvent) {
 583
 584                 if (taxon != null){
 585                         taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 586                 }
 587                 TaxonDescription taxonDescription = getTaxonDescription(taxon, ref, false, true);
 588                 taxonDescription.setTitleCache(ref.getTitleCache(), true);
 589                 taxon.addDescription(taxonDescription);
 590
 591                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
 592                 Feature feature = makeFeature(derivedUnitBase);
 593                 indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
 594                 indAssociation.setFeature(feature);
 595
 596                 for (Reference<?> citation : determinationEvent.getReferences()) {
 597                         indAssociation.addSource(DescriptionElementSource.NewInstance(null, null, citation, null));
 598                 }
 599
 600                 taxonDescription.addElement(indAssociation);
 601                 taxonDescription.setTaxon(taxon);
 602
 603                 getDescriptionService().saveOrUpdate(taxonDescription);
 604                 getTaxonService().saveOrUpdate(taxon);
 605         }
 606
 607         private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
 608                 if (unit.isInstanceOf(DerivedUnit.class)) {
 609                         return Feature.INDIVIDUALS_ASSOCIATION();
 610                 }
 611                 else if (unit.isInstanceOf(FieldObservation.class) || unit.isInstanceOf(Observation.class)) {
 612                         return Feature.OBSERVATION();
 613                 }
 614                 else if (unit.isInstanceOf(Fossil.class) || unit.isInstanceOf(LivingBeing.class) || unit.isInstanceOf(Specimen.class)) {
 615                         return Feature.SPECIMEN();
 616                 }
 617                 logger.warn("No feature defined for derived unit class: " + unit.getClass().getSimpleName());
 618                 return null;
 619         }
 620
 621         private void refreshTransaction(Abcd206ImportState state){
 622                 commitTransaction(state.getTx());
 623                 state.setTx(startTransaction());
 624                 try{
 625                         if (ref != null){
 626                                 ref = getReferenceService().find(ref.getUuid());
 627                         }
 628                         if (classification != null){
 629                                 classification = getClassificationService().find(classification.getUuid());
 630                         }
 631                         if (derivedUnitBase != null){
 632                                 derivedUnitBase = (DerivedUnitBase<?>) getOccurrenceService().find(derivedUnitBase.getUuid());
 633                         }
 634                 }
 635                 catch(Exception e){
 636                         state.setSuccess(false);
 637                         rollbackTransaction(state.getTx());
 638                         logger.error(e);
 639                         e.printStackTrace();
 640                 }
 641         }
 642
 643
 644     /**
 645      * getTaxon : search for an existing taxon in the database, for the same
 646      * reference
 647      *
 648      * @param config
 649      * @param scientificName
 650      * @param preferredFlag
 651      * @param i
 652      * @param taxonnametoinsert
 653      * @param preferredtaxonnametoinsert
 654      * @param taxonName
 655      * @return
 656      */
 657     private Taxon getTaxon(Abcd206ImportState state, String scientificName, int i, Rank rank, String nomenclature) {
 658         Abcd206ImportConfigurator config = state.getConfig();
 659
 660         if (rank == null) {
 661             System.out.println("getTaxon "+scientificName);
 662         } else {
 663             System.out.println("getTaxon "+scientificName+", "+rank.generateTitle());
 664         }
 665         Taxon taxon = null;
 666         NonViralName<?> taxonName = null;
 667
 668         if (config.isDoReUseTaxon()){
 669             List<TaxonBase> c = null;
 670             try {
 671                 Taxon cc= getTaxonService().findBestMatchingTaxon(scientificName);
 672                 if (cc != null && cc.getSec()!=null && cc.getSec().getTitleCache().equalsIgnoreCase(ref.getTitleCache())){
 673                     taxon=cc;
 674                 }
 675                 else{
 676                     c = getTaxonService().searchTaxaByName(scientificName, ref);
 677                     for (TaxonBase<?> b : c) {
 678                         taxon = (Taxon) b;
 679                     }
 680                 }
 681             } catch (Exception e) {
 682                 logger.info("Searchtaxabyname failed" + e);
 683                 taxon = null;
 684             }
 685         }
 686         if (!config.isDoReUseTaxon() || taxon == null){
 687             System.out.println("create new taxonName instance");
 688             if (config.isDoAutomaticParsing()){
 689                 taxonName = parseScientificName(scientificName);
 690             }
 691             else{
 692                 if (i>=0 && (dataHolder.atomisedIdentificationList != null || dataHolder.atomisedIdentificationList.size() > 0)) {
 693                     taxonName = setTaxonNameByType(dataHolder.atomisedIdentificationList.get(i), scientificName);
 694                 } else {
 695                     taxonName=null;
 696                 }
 697             }
 698             if(taxonName == null){
 699                 taxonName = NonViralName.NewInstance(rank);
 700                 taxonName.setFullTitleCache(scientificName,true);
 701                 taxonName.setTitleCache(scientificName, true);
 702             }
 703             System.out.println("ADD NEW TAXON *"+taxonName.getRank()+"*"+taxonName);
 704             if (rank != null && (taxonName.getRank() ==null || taxonName.getRank().toString().trim().isEmpty())) {
 705                 taxonName.setRank(rank);
 706             }
 707             getNameService().save(taxonName);
 708             taxon = Taxon.NewInstance(taxonName, ref); //sec set null
 709             getTaxonService().save(taxon);
 710             refreshTransaction(state);
 711            taxon= (Taxon) getTaxonService().find(taxon.getUuid());
 712         }
 713
 714
 715
 716         System.out.println("taxon.getUuid() suite :"+taxon.getUuid());
 717         return taxon;
 718     }
 719
 720
 721
 722         /**
 723          * getParentTaxon : get the taxonomic hierarchy for the current Taxon
 724          *
 725          * @param taxon
 726          * @param taxonName
 727          * @param originalName
 728          * @param config
 729          * @return a map with the parenttaxon and the parenttaxonname
 730          */
 731         private HashMap<Taxon, NonViralName<?>> getParentTaxon(Abcd206ImportState state, Taxon taxon, NonViralName<?> taxonName, NonViralName<?> originalName) {
 732
 733                 Abcd206ImportConfigurator config = state.getConfig();
 734                 Taxon parenttaxon = null;
 735                 NonViralName<?> parentName = null;
 736                 List<TaxonBase> c = null;
 737
 738                 List<String> highername = new ArrayList<String>();
 739                 Rank higherrank = null;
 740                 Rank taxonrank = taxonName.getRank();
 741
 742                 if(DEBUG) logger.info("getParentTaxon childname " + taxonName.getFullTitleCache() + ", rank " + taxonrank + ", originalname " + originalName.getFullTitleCache());
 743
 744                 HashMap<Taxon, NonViralName<?>> map = new HashMap<Taxon, NonViralName<?>>();
 745
 746
 747                 refreshTransaction(state);
 748
 749                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 750
 751                 if (taxonrank.isGenus()) {
 752                         // to change and add test DoReusetaxa
 753                         for (TaxonNode p : classification.getAllNodes()) {
 754                                 if(DEBUG) logger.info("p UUID "+p.getUuid().toString());
 755                                 if (classification.getTopmostNode(p.getTaxon()) == null) {
 756                                         if(DEBUG){
 757                                                 logger.info("taxon1 "+p.getTaxon().getTitleCache());
 758                                                 logger.info("taxon2 "+taxon.getTitleCache());
 759                                         }
 760                                         if (taxon.getTitleCache().contains(p.getTaxon().getTitleCache().split(SEC + ref)[0])) {
 761                                                 this.addParentChild(state, p.getTaxon(), taxon);
 762
 763                                                 refreshTransaction(state);
 764                                                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 765
 766                                                 break;
 767                                         }
 768                                 }
 769                         }
 770                         // add the genus to the root of the classification
 771                         TaxonNode p = this.addChildTaxon(state, taxon);
 772                         this.addChildNode(state, p);
 773
 774                         refreshTransaction(state);
 775
 776                         taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 777
 778                         return map;
 779                 }
 780
 781                 else if (taxonrank.isInfraGeneric()) {
 782                         if(DEBUG) logger.info("isInfrageneric");
 783                         highername.add(originalName.getGenusOrUninomial());
 784                         higherrank = Rank.GENUS();
 785                 }
 786
 787                 else if (taxonrank.isSpecies()) {
 788                         if(DEBUG) logger.info("isSpecies");
 789                         if (originalName.getGenusOrUninomial() != null) {
 790                                 highername.add(originalName.getGenusOrUninomial());
 791                                 higherrank = Rank.GENUS();
 792                         }
 793                         else {
 794                                 highername.add(originalName.getFullTitleCache().trim().split(" ")[0]);
 795                                 higherrank = Rank.GENUS();
 796                         }
 797                         if (originalName.getInfraGenericEpithet() != null) {
 798                                 highername.add(originalName.getInfraGenericEpithet());
 799                                 higherrank = Rank.INFRAGENUS();
 800                         }
 801                 }
 802
 803                 else if (taxonrank.isInfraSpecific()) {
 804                         if(DEBUG) logger.info("isInfraSpecies");
 805                         if (originalName.getGenusOrUninomial() != null){
 806                                 highername.add(originalName.getGenusOrUninomial());
 807                         }
 808                         if (originalName.getInfraGenericEpithet() != null) {
 809                                 highername.add(originalName.getInfraGenericEpithet());
 810                         }
 811                         if (originalName.getSpecificEpithet() != null){
 812                                 highername.add(originalName.getSpecificEpithet());
 813                         }
 814                         higherrank = Rank.SPECIES();
 815                 }
 816
 817                 String highernamestr = StringUtils.join(highername.iterator(), " ").split(SEC + ref.getTitleCache())[0].trim();
 818                 if(DEBUG) logger.info("higherNamest :: " + highernamestr);
 819                 if (config.isDoReUseTaxon() && highername.size() > 0 && isNotBlank(highernamestr)) {
 820                         boolean parentFound = false;
 821                         try {
 822                                 c = getTaxonService().searchTaxaByName(highernamestr, ref);
 823
 824                                 for (TaxonBase<?> b : c) {
 825                                         parenttaxon = (Taxon) b;
 826                                         Iterator<TaxonNode> it = parenttaxon.getTaxonNodes().iterator();
 827                                         // logger.warn("ICI2");
 828                                         while (it.hasNext()) {
 829                                                 TaxonNode tmpNode = it.next();
 830                                                 Taxon tmp = tmpNode.getTaxon();
 831                                                 if (tmp.getTitleCache().split(SEC + ref.getTitleCache())[0].trim().equalsIgnoreCase(highernamestr)) {
 832                                                         parenttaxon = tmp;
 833                                                         parentName = (NonViralName<?>) b.getName();
 834                                                         parentFound = true;
 835                                                 }
 836                                         }
 837                                         if(DEBUG){
 838                                                 if (!parentFound)
 839                                                         logger.info("parent not found");
 840                                                 else
 841                                                         logger.info("parent found: " + parenttaxon.getTitleCache());
 842                                         }
 843
 844                                 }
 845                         } catch (Exception e) {
 846                                 logger.info("Problem while trying to reuse existing taxon" + e);
 847                                 parenttaxon = null;
 848                         }
 849                         if (!parentFound)
 850                                 parenttaxon = null;
 851                 }
 852
 853                 if ((parenttaxon == null && highername.size() > 0 && isNotBlank(highernamestr)) || !config.isDoReUseTaxon()) {
 854                         // logger.info("ICI BIS");
 855                         parentName = NonViralName.NewInstance(null);
 856                         parentName.setFullTitleCache(highernamestr);
 857                         parentName.setNameCache(highernamestr);
 858                         parentName.setRank(higherrank);
 859                         if (higherrank == Rank.GENUS()){
 860                                 parentName.setGenusOrUninomial(highernamestr.split(" ")[0]);
 861                         }
 862
 863                         getNameService().save(parentName);
 864                         parenttaxon = Taxon.NewInstance(parentName, ref);
 865                         getTaxonService().save(parenttaxon);
 866
 867                 }
 868
 869                 map.put(parenttaxon, parentName);
 870                 return map;
 871         }
 872
 873         private TaxonNode addChildNode(Abcd206ImportState state, TaxonNode childNode){
 874                 TaxonNode re =null;
 875                 boolean exists=false;
 876                 if (state.getConfig().isDoReUseTaxon()){
 877                         Taxon taxon = childNode.getTaxon();
 878                         Set<TaxonNode> allNodes = classification.getAllNodes();
 879                         Taxon tmp;
 880                         if (allNodes.size()>0)
 881                                 for (TaxonNode tn:allNodes){
 882                                         tmp = tn.getTaxon();
 883                                         if (tmp.equals(taxon)){
 884                                                 re= tn;
 885                                                 exists=true;
 886                                                 break;
 887                                         }
 888                                 }
 889                 }
 890                 if(! state.getConfig().isDoReUseTaxon() || !exists){
 891                         refreshTransaction(state);
 892                         childNode = getTaxonNodeService().find(childNode.getUuid());
 893                         re= classification.addChildNode(childNode, ref, "", null);
 894                 }
 895                 return re;
 896         }
 897
 898         private void addParentChild(Abcd206ImportState state, Taxon parent, Taxon child){
 899                 if(DEBUG) logger.info("addParentChild");
 900                 boolean exists = false;
 901                 Taxon taxonFromHiber = null;
 902                 if (state.getConfig().isDoReUseTaxon()){
 903                         Set<TaxonNode> allNodes = classification.getAllNodes();
 904                         Taxon tmp;
 905                         if (allNodes.size()>0){
 906                                 for (TaxonNode tn:allNodes){
 907                                         tmp = tn.getTaxon();
 908                                         if (tmp.equals(parent)){
 909                                                 taxonFromHiber = tmp;
 910                                         }
 911                                 }
 912                         }
 913                         if (taxonFromHiber != null){
 914                                 Set<TaxonNode> children = taxonFromHiber.getTaxonNodes();
 915                                 if (children.size()>0)
 916                                         for (TaxonNode cn:children){
 917                                                 Taxon tmpt = cn.getTaxon();
 918                                                 if (tmpt.equals(child))
 919                                                         exists=true;
 920                                         }
 921                         }
 922                 }
 923                 if (! state.getConfig().isDoReUseTaxon() || !exists){
 924                         refreshTransaction(state);
 925                         parent = (Taxon) getTaxonService().find(parent.getUuid());
 926                         child = (Taxon) getTaxonService().find(child.getUuid());
 927                         classification.addParentChild(parent,child, ref, null);
 928                 }
 929         }
 930
 931         private TaxonNode addChildTaxon(Abcd206ImportState state,Taxon child){
 932                 TaxonNode re =null;
 933                 boolean exists=false;
 934                 if (state.getConfig().isDoReUseTaxon()){
 935                         Set<TaxonNode> allNodes = classification.getAllNodes();
 936                         Taxon tmp;
 937                         if (allNodes.size()>0){
 938                                 for (TaxonNode tn:allNodes){
 939                                         tmp = tn.getTaxon();
 940                                         if (tmp.equals(child)){
 941                                                 re = tn;
 942                                                 exists=true;
 943                                                 break;
 944                                         }
 945                                 }
 946                         }
 947                 }
 948                 if(! state.getConfig().isDoReUseTaxon() || !exists){
 949                         refreshTransaction(state);
 950                         child = (Taxon) getTaxonService().find(child.getUuid());
 951                         re= classification.addChildTaxon(child, ref, "", null);
 952                 }
 953                 return re;
 954         }
 955
 956
 957     /**
 958      * @param state
 959      * @param scientificName
 960      * @param parseInt
 961      * @return
 962      */
 963     private Taxon getTaxon(Abcd206ImportState state, String scientificName, int index, String nomenclature) {
 964         return getTaxon(state, scientificName, index, null, nomenclature);
 965     }
 966
 967     /**
 968      * HandleIdentifications : get the scientific names present in the ABCD
 969      * document and store link them with the observation/specimen data
 970      *
 971      * @param config
 972      */
 973     private void handleIdentifications(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
 974
 975         Abcd206ImportConfigurator config = state.getConfig();
 976
 977         String fullScientificNameString;
 978         Taxon taxon = null;
 979         Rank.GENUS();
 980         Rank.FAMILY();
 981
 982         String scientificName = "";
 983         boolean preferredFlag = false;
 984         boolean onePreferred = false;
 985
 986         List<String> scientificNames = new ArrayList<String>();
 987         if (dataHolder.nomenclatureCode == ""){
 988             dataHolder.nomenclatureCode = config.getNomenclaturalCode().toString();
 989         }
 990
 991         for (int i = 0; i < dataHolder.identificationList.size(); i++) {
 992
 993             fullScientificNameString = dataHolder.identificationList.get(i);
 994             fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
 995
 996             if (fullScientificNameString.indexOf(PREFERRED) != -1) {
 997                 scientificName = fullScientificNameString.split(PREFERRED)[0];
 998                 String pTmp = fullScientificNameString.split(PREFERRED)[1].split(CODE)[0];
 999                 if (pTmp.equals("1") || pTmp.toLowerCase().indexOf("true") != -1) {
1000                     preferredFlag = true;
1001                     onePreferred = true;
1002                 }
1003                 else {
1004                     preferredFlag = false;
1005                 }
1006             }
1007             else {
1008                 scientificName = fullScientificNameString;
1009             }
1010             if(DEBUG) {
1011                 logger.info("fullscientificname " + fullScientificNameString + ", *" + dataHolder.nomenclatureCode + "*");
1012             }
1013             if (fullScientificNameString.indexOf(CODE) != -1) {
1014                 if (fullScientificNameString.indexOf(':') != -1) {
1015                     dataHolder.nomenclatureCode = fullScientificNameString.split(CODE)[1].split(COLON)[1];
1016                 }
1017                 else{
1018                     dataHolder.nomenclatureCode = fullScientificNameString.split(CODE)[1];
1019                 }
1020             }
1021             scientificNames.add(scientificName+SPLITTER+preferredFlag+SPLITTER+i);
1022         }
1023         for (String name:scientificNames) {
1024             scientificName = name.split(SPLITTER)[0];
1025             String pref = name.split(SPLITTER)[1];
1026             String index = name.split(SPLITTER)[2];
1027             if (pref.equalsIgnoreCase("true") || scientificNames.size()==1) {
1028                 preferredFlag = true;
1029             } else {
1030                 preferredFlag =false;
1031             }
1032             taxon = getTaxon(state, scientificName,Integer.parseInt(index),dataHolder.nomenclatureCode);
1033             addTaxonNode(taxon, state,dataHolder.nomenclatureCode);
1034             taxon = (Taxon) getTaxonService().find(taxon.getUuid());
1035             linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
1036             //            refreshTransaction();
1037         }
1038         refreshTransaction(state);
1039         taxon = (Taxon) getTaxonService().find(taxon.getUuid());
1040     }
1041
1042     /**
1043      * @param taxon
1044      * @param config
1045      * @return
1046      */
1047     private void addTaxonNode(Taxon taxon, Abcd206ImportState state, String nomenclature) {
1048         logger.info("link taxon to a taxonNode");
1049         boolean exist = false;
1050         for (TaxonNode p : classification.getAllNodes()){
1051             if(p.getTaxon().equals(taxon)) {
1052                 exist =true;
1053             }
1054         }
1055         if (!exist){
1056             addParentTaxon(taxon, state,nomenclature);
1057         }
1058         refreshTransaction(state);
1059     }
1060
1061     private void addParentTaxon(Taxon taxon, Abcd206ImportState state,String nomenclature){
1062         System.out.println("addParentTaxon "+taxon.getTitleCache());
1063
1064                 NonViralName<?>  nvname = CdmBase.deproxy(taxon.getName(), NonViralName.class);
1065             Rank rank = nvname.getRank();
1066             Taxon genus =null;
1067             Taxon subgenus =null;
1068             Taxon species = null;
1069             Taxon subspecies = null;
1070             if (rank.isLower(Rank.GENUS() )){
1071                 String prefix = nvname.getGenusOrUninomial();
1072                 genus = getTaxon(state, prefix, -1, Rank.GENUS(),nomenclature);
1073                 saveOrUpdateClassification(null, genus);
1074             }
1075             if (rank.isLower(Rank.SUBGENUS())){
1076                 String prefix = nvname.getGenusOrUninomial();
1077                 String name = nvname.getInfraGenericEpithet();
1078                 if (name != null){
1079                     subgenus = getTaxon(state, prefix+" "+name, -1, Rank.SUBGENUS(),nomenclature);
1080                     saveOrUpdateClassification(genus, subgenus);
1081                 }
1082             }
1083             if (rank.isLower(Rank.SPECIES())){
1084                 if (subgenus!=null){
1085                     String prefix = nvname.getGenusOrUninomial();
1086                     String name = nvname.getInfraGenericEpithet();
1087                     String spe = nvname.getSpecificEpithet();
1088                     if (spe != null){
1089                         species = getTaxon(state, prefix+" "+name+" "+spe, -1, Rank.SPECIES(),nomenclature);
1090                         saveOrUpdateClassification(subgenus, species);
1091                     }
1092                 }
1093                 else{
1094                     String prefix = nvname.getGenusOrUninomial();
1095                     String name = nvname.getSpecificEpithet();
1096                     if (name != null){
1097                         species = getTaxon(state, prefix+" "+name, -1, Rank.SPECIES(),nomenclature);
1098                         saveOrUpdateClassification(genus, species);
1099                     }
1100                 }
1101             }
1102             if (rank.isInfraSpecific()){
1103                     subspecies = getTaxon(state, nvname.getFullTitleCache(), -1, Rank.SUBSPECIES(),nomenclature);
1104                     saveOrUpdateClassification(species, subspecies);
1105             }
1106         }
1107
1108     /**
1109      * @param currentTaxon
1110      * @param taxon
1111      */
1112     private void saveOrUpdateClassification(Taxon parent, Taxon child) {
1113         System.out.println("ADD CLASSIFICATION parent child "+parent+"," +child);
1114         if (parent != null) {
1115             parent = (Taxon) getTaxonService().find(parent.getUuid());
1116             child = (Taxon) getTaxonService().find(child.getUuid());
1117             classification.addParentChild(parent, child, ref, "");
1118         }
1119         if (parent == null) {
1120             child = (Taxon) getTaxonService().find(child.getUuid());
1121             classification.addChildTaxon(child, ref, "", null);
1122         }
1123         getClassificationService().saveOrUpdate(classification);
1124     }
1125
1126
1127
1128         private NonViralName<?> parseScientificName(String scientificName) {
1129                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1130                 NonViralName<?> taxonName = null;
1131                 boolean problem = false;
1132
1133                 if(DEBUG){
1134                         logger.info("parseScientificName " + dataHolder.nomenclatureCode.toString());
1135                 }
1136
1137                 if (dataHolder.nomenclatureCode.toString().equals("Zoological") || dataHolder.nomenclatureCode.toString().contains("ICZN")) {
1138                         taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
1139                         if (taxonName.hasProblem()) {
1140                                 problem = true;
1141                         }
1142                 }
1143                 if (dataHolder.nomenclatureCode.toString().equals("Botanical") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1144                         taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICBN, null);
1145                         if (taxonName.hasProblem()) {
1146                                 problem = true;
1147                         }
1148                 }
1149                 if (dataHolder.nomenclatureCode.toString().equals("Bacterial") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1150                         taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
1151                         if (taxonName.hasProblem()) {
1152                                 problem = true;
1153                         }
1154                 }
1155                 if (dataHolder.nomenclatureCode.toString().equals("Cultivar") || dataHolder.nomenclatureCode.toString().contains("ICNCP")) {
1156                         taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
1157                         if (taxonName.hasProblem()) {
1158                                 problem = true;
1159                         }
1160                 }
1161                 if (problem) {
1162                         logger.info("Parsing with problem in parseScientificName " + scientificName);
1163                         return null;
1164                 }
1165                 return taxonName;
1166
1167         }
1168
1169         private NonViralName<?> setTaxonNameByType(
1170                         HashMap<String, String> atomisedMap, String fullName) {
1171                 boolean problem = false;
1172                 if(DEBUG) logger.info("settaxonnamebytype " + dataHolder.nomenclatureCode.toString());
1173
1174                 if (dataHolder.nomenclatureCode.equals("Zoological")) {
1175                         NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
1176                         taxonName.setFullTitleCache(fullName, true);
1177                         taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1178                         taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1179                         taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
1180                         taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
1181
1182                         if (taxonName.getGenusOrUninomial() != null){
1183                                 taxonName.setRank(Rank.GENUS());
1184                         }
1185
1186                         else if (taxonName.getInfraGenericEpithet() != null){
1187                                 taxonName.setRank(Rank.SUBGENUS());
1188                         }
1189
1190                         else if (taxonName.getSpecificEpithet() != null){
1191                                 taxonName.setRank(Rank.SPECIES());
1192                         }
1193
1194                         else if (taxonName.getInfraSpecificEpithet() != null){
1195                                 taxonName.setRank(Rank.SUBSPECIES());
1196                         }
1197
1198                         Team team = null;
1199                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1200                                 team = Team.NewInstance();
1201                                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1202                         }
1203                         else {
1204                                 if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1205                                         team = Team.NewInstance();
1206                                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1207                                 }
1208                         }
1209                         if (team != null) {
1210                                 taxonName.setBasionymAuthorTeam(team);
1211                         }
1212                         else {
1213                                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1214                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1215                                 }
1216                                 else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1217                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
1218                                 }
1219                         }
1220                         if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1221                                 team = Team.NewInstance();
1222                                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1223                                 taxonName.setCombinationAuthorTeam(team);
1224                         }
1225                         if (taxonName.hasProblem()) {
1226                                 logger.info("pb ICZN");
1227                                 problem = true;
1228                         }
1229                         else {
1230                                 return taxonName;
1231                         }
1232                 }
1233                 else if (dataHolder.nomenclatureCode.equals("Botanical")) {
1234                         BotanicalName taxonName = (BotanicalName) parseScientificName(fullName);
1235                         if (taxonName != null){
1236                                 return taxonName;
1237                         }
1238                         else{
1239                                 taxonName = BotanicalName.NewInstance(null);
1240                         }
1241                         taxonName.setFullTitleCache(fullName, true);
1242                         taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1243                         taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
1244                         taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
1245                         try {
1246                                 taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
1247                         } catch (Exception e) {
1248                                 if (taxonName.getGenusOrUninomial() != null){
1249                                         taxonName.setRank(Rank.GENUS());
1250                                 }
1251                                 else if (taxonName.getInfraGenericEpithet() != null){
1252                                         taxonName.setRank(Rank.SUBGENUS());
1253                                 }
1254                                 else if (taxonName.getSpecificEpithet() != null){
1255                                         taxonName.setRank(Rank.SPECIES());
1256                                 }
1257                                 else if (taxonName.getInfraSpecificEpithet() != null){
1258                                         taxonName.setRank(Rank.SUBSPECIES());
1259                                 }
1260                         }
1261                         Team team = null;
1262                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1263                                 team = Team.NewInstance();
1264                                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1265                                 taxonName.setBasionymAuthorTeam(team);
1266                         }
1267                         if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1268                                 team = Team.NewInstance();
1269                                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
1270                                 taxonName.setCombinationAuthorTeam(team);
1271                         }
1272                         if (team == null) {
1273                                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1274                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1275                                 }
1276                                 else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1277                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
1278                                 }
1279                         }
1280                         if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1281                                 team = Team.NewInstance();
1282                                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1283                                 taxonName.setCombinationAuthorTeam(team);
1284                         }
1285                         if (taxonName.hasProblem()) {
1286                                 logger.info("pb ICBN");
1287                                 problem = true;
1288                         }
1289                         else {
1290                                 return taxonName;
1291                         }
1292                 }
1293                 else if (dataHolder.nomenclatureCode.equals("Bacterial")) {
1294                         NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
1295                         taxonName.setFullTitleCache(fullName, true);
1296                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
1297                         taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1298                         taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
1299                         taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
1300
1301                         if (taxonName.getGenusOrUninomial() != null){
1302                                 taxonName.setRank(Rank.GENUS());
1303                         }
1304                         else if (taxonName.getInfraGenericEpithet() != null){
1305                                 taxonName.setRank(Rank.SUBGENUS());
1306                         }
1307                         else if (taxonName.getSpecificEpithet() != null){
1308                                 taxonName.setRank(Rank.SPECIES());
1309                         }
1310                         else if (taxonName.getInfraSpecificEpithet() != null){
1311                                 taxonName.setRank(Rank.SUBSPECIES());
1312                         }
1313
1314                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1315                                 Team team = Team.NewInstance();
1316                                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1317                                 taxonName.setCombinationAuthorTeam(team);
1318                         }
1319                         if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
1320                                 Team team = Team.NewInstance();
1321                                 team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
1322                                 taxonName.setBasionymAuthorTeam(team);
1323                         }
1324                         if (taxonName.hasProblem()) {
1325                                 logger.info("pb ICNB");
1326                                 problem = true;
1327                         }
1328                         else {
1329                                 return taxonName;
1330                         }
1331                 }
1332                 else if (dataHolder.nomenclatureCode.equals("Cultivar")) {
1333                         CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
1334
1335                         if (taxonName.hasProblem()) {
1336                                 logger.info("pb ICNCP");
1337                                 problem = true;
1338                         }
1339                         else {
1340                                 return taxonName;
1341                         }
1342                         return taxonName;
1343                 }
1344
1345                 if (problem) {
1346                         logger.info("Problem im setTaxonNameByType ");
1347                         NonViralName<?> taxonName = NonViralName.NewInstance(null);
1348                         taxonName.setFullTitleCache(fullName, true);
1349                         return taxonName;
1350                 }
1351                 NonViralName<?> tn = NonViralName.NewInstance(null);
1352                 return tn;
1353         }
1354
1355         private String getFromMap(HashMap<String, String> atomisedMap, String key) {
1356                 String value = null;
1357                 if (atomisedMap.containsKey(key)) {
1358                         value = atomisedMap.get(key);
1359                 }
1360
1361                 try {
1362                         if (value != null && key.matches(".*Year.*")) {
1363                                 value = value.trim();
1364                                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
1365                                         String tmp = value.split("[0-9]{4}$")[0];
1366                                         int year = Integer.parseInt(value.split(tmp)[1]);
1367                                         if (year >= 1752) {
1368                                                 value = tmp;
1369                                         }
1370                                         else {
1371                                                 value = null;
1372                                         }
1373                                 }
1374                                 else {
1375                                         value = null;
1376                                 }
1377                         }
1378                 }
1379                 catch (Exception e) {
1380                         value = null;
1381                 }
1382                 return value;
1383         }
1384
1385         private void compareABCDtoCDM(URI urlFileName, List<String> knownElts, Abcd206XMLFieldGetter abcdFieldGetter) {
1386                 try {
1387                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
1388                         DocumentBuilder constructeur = factory.newDocumentBuilder();
1389                         URL url = urlFileName.toURL();
1390                         Object o = url.getContent();
1391                         InputStream is = (InputStream) o;
1392                         Document document = constructeur.parse(is);
1393                         Element root = document.getDocumentElement();
1394                         abcdFieldGetter.traverse(root);
1395                 }
1396                 catch (ParserConfigurationException e){
1397                         e.printStackTrace();
1398                 }
1399                 catch (SAXException e) {
1400                         e.printStackTrace();
1401                 }
1402                 catch (IOException e) {
1403                         e.printStackTrace();
1404                 }
1405                 Set<String> elts = dataHolder.allABCDelements.keySet();
1406                 Iterator<String> it = elts.iterator();
1407                 String elt;
1408                 while (it.hasNext()) {
1409                         elt = it.next();
1410                         if (knownElts.indexOf(elt) == -1) {
1411                                 if(DEBUG) logger.info("Unmerged ABCD element: " + elt + " - "+ dataHolder.allABCDelements.get(elt));
1412                         }
1413                 }
1414         }
1415
1416     /**
1417      * @param unitsList
1418      * @param state
1419      */
1420     private void prepareCollectors(Abcd206ImportState state, NodeList unitsList, Abcd206XMLFieldGetter abcdFieldGetter) {
1421         List<String> collectors = new ArrayList<String>();
1422         List<String> teams = new ArrayList<String>();
1423         List<List<String>> collectorinteams = new ArrayList<List<String>>();
1424
1425         for (int i = 0; i < unitsList.getLength(); i++) {
1426             this.getCollectorsFromXML((Element) unitsList.item(i), abcdFieldGetter);
1427             for (String agent : dataHolder.gatheringAgentList) {
1428                 collectors.add(agent);
1429             }
1430             List<String> tmpTeam = new ArrayList<String>(new HashSet<String>(dataHolder.gatheringTeamList));
1431             if(!tmpTeam.isEmpty()) {
1432                 teams.add(StringUtils.join(tmpTeam.toArray()," & "));
1433             }
1434             for (String agent:tmpTeam) {
1435                 collectors.add(agent);
1436             }
1437         }
1438
1439         List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
1440         List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
1441
1442
1443         //existing teams in DB
1444         Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
1445         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
1446
1447         Set<UUID> uuids = new HashSet<UUID>();
1448         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
1449             uuids.add(hibernateT.getUuid());
1450         }
1451         if (!uuids.isEmpty()){
1452             List<AgentBase> existingTeams = getAgentService().find(uuids);
1453             for (AgentBase<?> existingP:existingTeams){
1454                 titleCacheTeam.put(existingP.getTitleCache(),(Team) existingP);
1455             }
1456         }
1457
1458
1459         Map<String,UUID> teamMap = new HashMap<String, UUID>();
1460         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
1461             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
1462         }
1463
1464         //existing persons in DB
1465         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
1466         Map<String,Person> titleCachePerson = new HashMap<String, Person>();
1467         uuids = new HashSet<UUID>();
1468         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
1469             uuids.add(hibernateP.getUuid());
1470         }
1471
1472         if (!uuids.isEmpty()){
1473             List<AgentBase> existingPersons = getAgentService().find(uuids);
1474             for (AgentBase<?> existingP:existingPersons){
1475                 titleCachePerson.put(existingP.getTitleCache(),(Person) existingP);
1476             }
1477         }
1478
1479         Map<String,UUID> personMap = new HashMap<String, UUID>();
1480         for (UuidAndTitleCache<Person> person:hiberPersons){
1481             personMap.put(person.getTitleCache(), person.getUuid());
1482         }
1483
1484         java.util.Collection<AgentBase> personToadd = new ArrayList<AgentBase>();
1485         java.util.Collection<AgentBase> teamToAdd = new ArrayList<AgentBase>();
1486
1487         for (String collector:collectorsU){
1488             Person p = Person.NewInstance();
1489             p.setTitleCache(collector,true);
1490             if (!personMap.containsKey(p.getTitleCache())){
1491                 personToadd.add(p);
1492             }
1493         }
1494         for (String team:teamsU){
1495             Team p = Team.NewInstance();
1496             p.setTitleCache(team,true);
1497             if (!teamMap.containsKey(p.getTitleCache())){
1498                 teamToAdd.add(p);
1499             }
1500         }
1501
1502         if(!personToadd.isEmpty()){
1503             Map<UUID, AgentBase> uuuidPerson = getAgentService().save(personToadd);
1504             for (UUID u:uuuidPerson.keySet()){
1505                 titleCachePerson.put(uuuidPerson.get(u).getTitleCache(),(Person) uuuidPerson.get(u) );
1506             }
1507         }
1508
1509         Person ptmp ;
1510         Map <String,Integer>teamdone = new HashMap<String, Integer>();
1511         for (List<String> collteam: collectorinteams){
1512             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
1513                 Team team = new Team();
1514                 boolean em =true;
1515                 for (String collector:collteam){
1516                     ptmp = Person.NewInstance();
1517                     ptmp.setTitleCache(collector,true);
1518                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
1519                     team.addTeamMember(p2);
1520                     em=false;
1521                 }
1522                 if (!em) {
1523                     teamToAdd.add(team);
1524                 }
1525                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
1526             }
1527         }
1528
1529         if(!teamToAdd.isEmpty()){
1530             Map<UUID, AgentBase> uuuidTeam =  getAgentService().save(teamToAdd);
1531             for (UUID u:uuuidTeam.keySet()){
1532                 titleCacheTeam.put(uuuidTeam.get(u).getTitleCache(), (Team) uuuidTeam.get(u) );
1533             }
1534         }
1535
1536         state.getConfig().setTeams(titleCacheTeam);
1537         state.getConfig().setPersons(titleCachePerson);
1538     }
1539
1540         @Override
1541         protected boolean isIgnore(Abcd206ImportState state) {
1542                 return false;
1543         }
1544
1545
1546 }