cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.net.MalformedURLException;
  15 import java.net.URI;
  16 import java.net.URL;
  17 import java.util.ArrayList;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24 import java.util.UUID;
  25
  26 import javax.xml.parsers.DocumentBuilder;
  27 import javax.xml.parsers.DocumentBuilderFactory;
  28 import javax.xml.parsers.ParserConfigurationException;
  29
  30 import org.apache.commons.lang.StringUtils;
  31 import org.apache.log4j.Logger;
  32 import org.springframework.stereotype.Component;
  33 import org.w3c.dom.Document;
  34 import org.w3c.dom.Element;
  35 import org.w3c.dom.NodeList;
  36 import org.xml.sax.SAXException;
  37
  38 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  39 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  40 import eu.etaxonomy.cdm.io.specimen.SpecimenImportBase;
  41 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  42 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  43 import eu.etaxonomy.cdm.model.agent.AgentBase;
  44 import eu.etaxonomy.cdm.model.agent.Institution;
  45 import eu.etaxonomy.cdm.model.agent.Person;
  46 import eu.etaxonomy.cdm.model.agent.Team;
  47 import eu.etaxonomy.cdm.model.common.CdmBase;
  48 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  49 import eu.etaxonomy.cdm.model.common.Language;
  50 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  51 import eu.etaxonomy.cdm.model.common.UuidAndTitleCache;
  52 import eu.etaxonomy.cdm.model.description.Feature;
  53 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  54 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  55 import eu.etaxonomy.cdm.model.location.NamedArea;
  56 import eu.etaxonomy.cdm.model.media.Media;
  57 import eu.etaxonomy.cdm.model.name.BacterialName;
  58 import eu.etaxonomy.cdm.model.name.BotanicalName;
  59 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  60 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  61 import eu.etaxonomy.cdm.model.name.NonViralName;
  62 import eu.etaxonomy.cdm.model.name.Rank;
  63 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  64 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  65 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  66 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  67 import eu.etaxonomy.cdm.model.occurrence.Collection;
  68 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  69 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
  70 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  71 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
  72 import eu.etaxonomy.cdm.model.occurrence.Fossil;
  73 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  74 import eu.etaxonomy.cdm.model.occurrence.LivingBeing;
  75 import eu.etaxonomy.cdm.model.occurrence.Observation;
  76 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  77 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  78 import eu.etaxonomy.cdm.model.reference.Reference;
  79 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  80 import eu.etaxonomy.cdm.model.taxon.Classification;
  81 import eu.etaxonomy.cdm.model.taxon.Taxon;
  82 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  83 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  84 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  85
  86 /**
  87  * @author p.kelbert
  88  * @created 20.10.2008
  89  * @version 1.0
  90  */
  91 @Component
  92 public class Abcd206Import extends SpecimenImportBase<Abcd206ImportConfigurator, Abcd206ImportState> {
  93         private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  94
  95
  96         private final boolean DEBUG = false;
  97
  98         private static final String SEC = "sec. ";
  99         private static final String PREFERRED = "_preferred_";
 100         private static final String CODE = "_code_";
 101         private static final String COLON = ":";
 102         private static final String SPLITTER = "--";
 103
 104
 105         private static String prefix = "";
 106
 107         //TODO make all fields ABCD206ImportState variables
 108         private Classification classification = null;
 109         private Reference<?> ref = null;
 110
 111         private Abcd206DataHolder dataHolder;
 112         private DerivedUnitBase<?> derivedUnitBase;
 113
 114         public Abcd206Import() {
 115                 super();
 116         }
 117
 118         @Override
 119         protected boolean doCheck(Abcd206ImportState state) {
 120                 logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
 121                 return true;
 122         }
 123
 124         /**
 125          * getClassification : get the classification declared in the ImportState
 126          *
 127          * @param state
 128          * @return
 129          */
 130         private void setClassification(Abcd206ImportState state) {
 131                 if (classification == null) {
 132                         String name = NB(state.getConfig().getClassificationName());
 133
 134                         classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 135                         if (state.getConfig().getClassificationUuid() != null) {
 136                                 classification.setUuid(state.getConfig().getClassificationUuid());
 137                         }
 138                         getClassificationService().saveOrUpdate(classification);
 139                         refreshTransaction(state);
 140                 }
 141         }
 142
 143
 144         @Override
 145         public void doInvoke(Abcd206ImportState state) {
 146                 state.setTx(startTransaction());
 147
 148                 logger.info("INVOKE Specimen Import from ABCD2.06 XML ");
 149                 URI sourceName = state.getConfig().getSource();
 150                 NodeList unitsList = getUnitsNodeList(sourceName);
 151
 152                 ref = state.getConfig().getSourceReference();
 153                 setClassification(state);
 154
 155                 if (unitsList != null) {
 156                         String message = "nb units to insert: " + unitsList.getLength();
 157                         logger.info(message);
 158                         updateProgress(state, message);
 159                         dataHolder = new Abcd206DataHolder();
 160
 161                         Abcd206XMLFieldGetter abcdFieldGetter = new Abcd206XMLFieldGetter(dataHolder, prefix);
 162
 163                         prepareCollectors(state, unitsList, abcdFieldGetter);
 164
 165                         for (int i = 0; i < unitsList.getLength(); i++) {
 166
 167                                 this.setUnitPropertiesXML( (Element) unitsList.item(i), abcdFieldGetter);
 168 //                              refreshTransaction(state);
 169                                 this.handleSingleUnit(state, i);
 170
 171                                 // compare the ABCD elements added in to the CDM and the
 172                                 // unhandled ABCD elements
 173                                 //compareABCDtoCDM(sourceName, dataHolder.knownABCDelements, abcdFieldGetter);
 174
 175                                 // reset the ABCD elements added in CDM
 176                                 // knownABCDelements = new ArrayList<String>();
 177                                 dataHolder.allABCDelements = new HashMap<String, String>();
 178
 179                                 //refreshTransaction(state);
 180                         }
 181                 }
 182                 commitTransaction(state.getTx());
 183                 return;
 184
 185         }
 186
 187         /*
 188          * Return the list of root nodes for an ABCD 2.06 XML file
 189          *
 190          * @param fileName: the file's location
 191          *
 192          * @return the list of root nodes ("Unit")
 193          */
 194         protected NodeList getUnitsNodeList(URI urlFileName) {
 195                 NodeList unitList = null;
 196                 try {
 197                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 198                         DocumentBuilder builder = factory.newDocumentBuilder();
 199                         URL url = urlFileName.toURL();
 200                         Object o = url.getContent();
 201                         InputStream is = (InputStream) o;
 202                         Document document = builder.parse(is);
 203                         Element root = document.getDocumentElement();
 204                         unitList = root.getElementsByTagName("Unit");
 205                         if (unitList.getLength() == 0) {
 206                                 unitList = root.getElementsByTagName("abcd:Unit");
 207                                 prefix = "abcd:";
 208                         }
 209                 } catch (Exception e) {
 210                         logger.warn(e);
 211                 }
 212                 return unitList;
 213         }
 214
 215         /*
 216          * Stores the unit with its Gathering informations in the CDM
 217          */
 218         private void handleSingleUnit(Abcd206ImportState state, int i) {
 219                 logger.info("handleSingleUnit");
 220
 221                 try {
 222                         updateProgress(state, "Importing data for unit: " + dataHolder.unitID);
 223
 224
 225                         // create facade
 226                         DerivedUnitFacade derivedUnitFacade = getFacade();
 227                         derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 228
 229                         /**
 230                          * GATHERING EVENT
 231                          */
 232
 233                         // gathering event
 234             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(getTermService(), dataHolder.locality, dataHolder.languageIso,
 235                     dataHolder.longitude, dataHolder.latitude, dataHolder.gatheringAgentList, dataHolder.gatheringTeamList, state.getConfig());
 236
 237                         // country
 238                         UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea(NB(dataHolder.isocountry), NB(dataHolder.country), this);
 239                         NamedArea areaCountry = unitsGatheringArea.getArea();
 240
 241                         // other areas
 242                         unitsGatheringArea = new UnitsGatheringArea(dataHolder.namedAreaList);
 243                         ArrayList<NamedArea> otherAreas = unitsGatheringArea.getAreas();
 244                         for (NamedArea namedArea : otherAreas) {
 245                                 unitsGatheringEvent.addArea(namedArea);
 246                         }
 247
 248                         // copy gathering event to facade
 249                         GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 250                         derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 251                         derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 252                         derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 253                         derivedUnitFacade.setCountry(areaCountry);
 254                         derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 255
 256                         // TODO exsiccatum
 257
 258                         // add fieldNumber
 259                         derivedUnitFacade.setFieldNumber(NB(dataHolder.fieldNumber));
 260
 261                         // //add Multimedia URLs
 262                         if (dataHolder.multimediaObjects.size() != -1) {
 263                                 for (String multimediaObject : dataHolder.multimediaObjects) {
 264                                         Media media;
 265                                         try {
 266                                                 media = getImageMedia(multimediaObject, READ_MEDIA_DATA, false);
 267                                                 derivedUnitFacade.addDerivedUnitMedia(media);
 268                                         } catch (MalformedURLException e) {
 269                                                 // TODO Auto-generated catch block
 270                                                 e.printStackTrace();
 271                                         }
 272
 273                                 }
 274                         }
 275
 276                         /*
 277                          * merge AND STORE DATA
 278                          */
 279                         getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 280
 281                         for (NamedArea area : otherAreas) {
 282                                 getTermService().saveOrUpdate(area);// merge it sooner (foreach area)
 283                         }
 284                         getTermService().saveLanguageData(unitsGatheringEvent.getLocality());
 285
 286                         // handle collection data
 287                         setCollectionData(state.getConfig(), derivedUnitFacade);
 288
 289
 290                         getOccurrenceService().saveOrUpdate(derivedUnitBase);
 291                         refreshTransaction(state);
 292
 293                         // handle identifications
 294                         handleIdentifications(state, derivedUnitFacade);
 295
 296                         logger.info("saved ABCD specimen ...");
 297
 298                 } catch (Exception e) {
 299                         logger.warn("Error when reading record!!");
 300                         e.printStackTrace();
 301                         state.setUnsuccessfull();
 302                 }
 303
 304                 return;
 305         }
 306
 307         /**
 308          * setCollectionData : store the collection object into the
 309          * derivedUnitFacade
 310          *
 311          * @param config
 312          */
 313         private void setCollectionData(Abcd206ImportConfigurator config,
 314                         DerivedUnitFacade derivedUnitFacade) {
 315                 // set catalogue number (unitID)
 316                 derivedUnitFacade.setCatalogNumber(NB(dataHolder.unitID));
 317                 derivedUnitFacade.setAccessionNumber(NB(dataHolder.accessionNumber));
 318                 // derivedUnitFacade.setCollectorsNumber(NB(dataHolder.collectorsNumber));
 319
 320                 /*
 321                  * INSTITUTION & COLLECTION
 322                  */
 323                 // manage institution
 324                 Institution institution = this.getInstitution(NB(dataHolder.institutionCode), config);
 325                 // manage collection
 326                 Collection collection = this.getCollection(institution, config);
 327                 // link specimen & collection
 328                 derivedUnitFacade.setCollection(collection);
 329         }
 330
 331         /**
 332          * getFacade : get the DerivedUnitFacade based on the recordBasis
 333          *
 334          * @return DerivedUnitFacade
 335          */
 336         private DerivedUnitFacade getFacade() {
 337         if(DEBUG) {
 338             logger.info("getFacade()");
 339         }
 340                 DerivedUnitType type = null;
 341
 342                 // create specimen
 343                 if (NB((dataHolder.recordBasis)) != null) {
 344                         if (dataHolder.recordBasis.toLowerCase().startsWith("s") || dataHolder.recordBasis.toLowerCase().contains("specimen")) {// specimen
 345                                 type = DerivedUnitType.Specimen;
 346                         }
 347                         if (dataHolder.recordBasis.toLowerCase().startsWith("o")) {
 348                                 type = DerivedUnitType.Observation;
 349                         }
 350                         if (dataHolder.recordBasis.toLowerCase().contains("fossil")){
 351                                 type = DerivedUnitType.Fossil;
 352                         }
 353                         if (dataHolder.recordBasis.toLowerCase().startsWith("l")) {
 354                                 type = DerivedUnitType.LivingBeing;
 355                         }
 356                         if (type == null) {
 357                                 logger.info("The basis of record does not seem to be known: " + dataHolder.recordBasis);
 358                                 type = DerivedUnitType.DerivedUnit;
 359                         }
 360                         // TODO fossils?
 361                 } else {
 362                         logger.info("The basis of record is null");
 363                         type = DerivedUnitType.DerivedUnit;
 364                 }
 365                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 366                 return derivedUnitFacade;
 367         }
 368
 369     private void getCollectorsFromXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 370         NodeList group;
 371
 372         group = root.getChildNodes();
 373         for (int i = 0; i < group.getLength(); i++) {
 374             if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 375                 group = group.item(i).getChildNodes();
 376                 break;
 377             }
 378         }
 379         dataHolder.gatheringAgentList = new ArrayList<String>();
 380         dataHolder.gatheringTeamList = new ArrayList<String>();
 381         abcdFieldGetter.getType(root);
 382         abcdFieldGetter.getGatheringPeople(root);
 383     }
 384
 385         /**
 386          * Store the unit's properties into variables Look which unit is the
 387          * preferred one Look what kind of name it is supposed to be, for the
 388          * parsing (Botanical, Zoological)
 389          *
 390          * @param racine: the root node for a single unit
 391          */
 392         private void setUnitPropertiesXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter) {
 393                 try {
 394                         NodeList group;
 395
 396                         group = root.getChildNodes();
 397                         for (int i = 0; i < group.getLength(); i++) {
 398                                 if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
 399                                         group = group.item(i).getChildNodes();
 400                                         break;
 401                                 }
 402                         }
 403                         dataHolder.identificationList = new ArrayList<String>();
 404                         dataHolder.statusList = new ArrayList<SpecimenTypeDesignationStatus>();
 405                         dataHolder.atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
 406                         dataHolder.referenceList = new ArrayList<String>();
 407                         dataHolder.multimediaObjects = new ArrayList<String>();
 408
 409                         abcdFieldGetter.getScientificNames(group);
 410                         abcdFieldGetter.getType(root);
 411
 412             if(DEBUG) {
 413                 logger.info("this.identificationList "+dataHolder.identificationList.toString());
 414             }
 415             abcdFieldGetter.getIDs(root);
 416                         abcdFieldGetter.getRecordBasis(root);
 417                         abcdFieldGetter.getMultimedia(root);
 418                         abcdFieldGetter.getNumbers(root);
 419                         abcdFieldGetter.getGeolocation(root);
 420                         abcdFieldGetter.getGatheringPeople(root);
 421                         boolean referencefound = abcdFieldGetter.getReferences(root);
 422                         if (!referencefound) {
 423                                 dataHolder.referenceList.add(ref.getTitleCache());
 424                         }
 425
 426                 } catch (Exception e) {
 427                         logger.info("Error occured while parsing XML file" + e);
 428                 }
 429         }
 430
 431         private Institution getInstitution(String institutionCode, Abcd206ImportConfigurator config) {
 432                 Institution institution;
 433                 List<Institution> institutions;
 434                 try {
 435                         if(DEBUG){ logger.info(dataHolder.institutionCode);}
 436                         institutions = getAgentService().searchInstitutionByCode(dataHolder.institutionCode);
 437                 } catch (Exception e) {
 438                         institutions = new ArrayList<Institution>();
 439                 }
 440                 if (institutions.size() == 0 || !config.isReUseExistingMetadata()) {
 441                         if(DEBUG) {
 442                                 logger.info("Institution (agent) unknown or not allowed to reuse existing metadata");
 443                         }
 444                         // create institution
 445                         institution = Institution.NewInstance();
 446                         institution.setCode(NB(dataHolder.institutionCode));
 447                 }
 448                 else {
 449                         if(DEBUG){
 450                                 logger.info("Institution (agent) already in the db");
 451                         }
 452                         institution = institutions.get(0);
 453                 }
 454                 if(DEBUG) logger.info("getinstitution " + institution.toString());
 455                 return institution;
 456         }
 457
 458         /**
 459          * Look if the Collection does already exists
 460          * @param collectionCode: a string
 461          * @param institution: the current Institution
 462          * @param app
 463          * @return the Collection (existing or new)
 464          */
 465         private Collection getCollection(Institution institution, Abcd206ImportConfigurator config) {
 466                 Collection collection = Collection.NewInstance();
 467                 List<Collection> collections;
 468                 try {
 469                         collections = getCollectionService().searchByCode(dataHolder.collectionCode);
 470                 } catch (Exception e) {
 471                         collections = new ArrayList<Collection>();
 472                 }
 473                 if (collections.size() == 0 || !config.isReUseExistingMetadata()) {
 474                         if(DEBUG) logger.info("Collection not found or do not reuse existing metadata  " + dataHolder.collectionCode);
 475                         // create new collection
 476                         collection.setCode(NB(dataHolder.collectionCode));
 477                         collection.setCodeStandard(NB("GBIF"));
 478                         collection.setInstitute(institution);
 479                 } else {
 480                         boolean collectionFound = false;
 481                         for (int i = 0; i < collections.size(); i++) {
 482                                 collection = collections.get(i);
 483                                 try {
 484                                         if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())) {
 485                                                 // found a collection with the same code and the same institution
 486                                                 collectionFound = true;
 487                                                 break;
 488                                         }
 489                                 } catch (NullPointerException e) {
 490                                         //TODO: exception?
 491                                 }
 492                         }
 493                         if (!collectionFound) {
 494                                 collection.setCode(NB(dataHolder.collectionCode));
 495                                 collection.setCodeStandard(NB("GBIF"));
 496                                 collection.setInstitute(institution);
 497                         }
 498
 499                 }
 500                 return collection;
 501         }
 502
 503         /**
 504          * join DeterminationEvent to the Taxon Object
 505          * @param taxon current Taxon Object
 506          * @param preferredFlag preferred name, boolean
 507          * @param config current ABCD Import configurator
 508          */
 509
 510         private void linkDeterminationEvent(Abcd206ImportState state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
 511                 Abcd206ImportConfigurator config = state.getConfig();
 512
 513                 if(DEBUG){
 514                         logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
 515                 }
 516
 517 //              refreshTransaction(state);
 518
 519                 if (taxon != null){
 520                         taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 521                 }
 522
 523
 524                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 525                 determinationEvent.setTaxon(taxon);
 526                 determinationEvent.setPreferredFlag(preferredFlag);
 527
 528                 determinationEvent.setIdentifiedUnit(derivedUnitBase);
 529
 530                 derivedUnitBase.addDetermination(determinationEvent);
 531 //              refreshTransaction(state);
 532
 533                 try {
 534                         if(DEBUG){
 535                                 logger.info("NB TYPES INFO: "+ dataHolder.statusList.size());
 536                         }
 537                         for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : dataHolder.statusList) {
 538                                 if (specimenTypeDesignationstatus != null) {
 539                                         if(DEBUG){
 540                                                 logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
 541                                         }
 542                                         if (taxon != null){
 543                                                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 544                                         }
 545                                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) getTermService().find(specimenTypeDesignationstatus.getUuid());
 546                                         //Designation
 547                                         TaxonNameBase<?,?> name = taxon.getName();
 548                                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
 549
 550                                         designation.setTypeStatus(specimenTypeDesignationstatus);
 551                                         designation.setTypeSpecimen(derivedUnitBase);
 552                                         name.addTypeDesignation(designation, true);
 553                                         refreshTransaction(state);
 554
 555                                 }
 556                         }
 557                 } catch (Exception e) {
 558                         logger.warn("PB addding SpecimenType " + e);
 559                 }
 560
 561                 for (String strReference : dataHolder.referenceList) {
 562                         if (isNotBlank(strReference)){
 563                                 Reference<?> reference = ReferenceFactory.newGeneric();
 564                                 reference.setTitleCache(strReference, true);
 565                                 getReferenceService().saveOrUpdate(reference);
 566
 567                                 determinationEvent.addReference(reference);
 568                         }
 569                 }
 570
 571                 getOccurrenceService().saveOrUpdate(derivedUnitBase);
 572                 refreshTransaction(state);
 573
 574
 575                 if (config.isDoCreateIndividualsAssociations()) {
 576                         if(DEBUG){ logger.info("isDoCreateIndividualsAssociations");}
 577
 578                    makeIndividualsAssociation(state, taxon, determinationEvent);
 579                    getOccurrenceService().saveOrUpdate(derivedUnitBase);
 580                 }
 581         }
 582
 583         private void makeIndividualsAssociation(Abcd206ImportState state, Taxon taxon, DeterminationEvent determinationEvent) {
 584
 585                 if (taxon != null){
 586                         taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 587                 }
 588                 TaxonDescription taxonDescription = getTaxonDescription(taxon, ref, false, true);
 589                 taxonDescription.setTitleCache(ref.getTitleCache(), true);
 590                 taxon.addDescription(taxonDescription);
 591
 592                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
 593                 Feature feature = makeFeature(derivedUnitBase);
 594                 indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
 595                 indAssociation.setFeature(feature);
 596
 597                 for (Reference<?> citation : determinationEvent.getReferences()) {
 598                         indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.PrimaryTaxonomicSource,
 599                                         null, null, citation, null));
 600                 }
 601
 602                 taxonDescription.addElement(indAssociation);
 603                 taxonDescription.setTaxon(taxon);
 604
 605                 getDescriptionService().saveOrUpdate(taxonDescription);
 606                 getTaxonService().saveOrUpdate(taxon);
 607         }
 608
 609         private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
 610                 if (unit.isInstanceOf(DerivedUnit.class)) {
 611                         return Feature.INDIVIDUALS_ASSOCIATION();
 612                 }
 613                 else if (unit.isInstanceOf(FieldObservation.class) || unit.isInstanceOf(Observation.class)) {
 614                         return Feature.OBSERVATION();
 615                 }
 616                 else if (unit.isInstanceOf(Fossil.class) || unit.isInstanceOf(LivingBeing.class) || unit.isInstanceOf(Specimen.class)) {
 617                         return Feature.SPECIMEN();
 618                 }
 619                 logger.warn("No feature defined for derived unit class: " + unit.getClass().getSimpleName());
 620                 return null;
 621         }
 622
 623         private void refreshTransaction(Abcd206ImportState state){
 624                 commitTransaction(state.getTx());
 625                 state.setTx(startTransaction());
 626                 try{
 627                         if (ref != null){
 628                                 ref = getReferenceService().find(ref.getUuid());
 629                         }
 630                         if (classification != null){
 631                                 classification = getClassificationService().find(classification.getUuid());
 632                         }
 633                         if (derivedUnitBase != null){
 634                                 derivedUnitBase = (DerivedUnitBase<?>) getOccurrenceService().find(derivedUnitBase.getUuid());
 635                         }
 636                 }
 637                 catch(Exception e){
 638                         state.setSuccess(false);
 639                         rollbackTransaction(state.getTx());
 640                         logger.error(e);
 641                         e.printStackTrace();
 642                 }
 643         }
 644
 645
 646     /**
 647      * getTaxon : search for an existing taxon in the database, for the same
 648      * reference
 649      *
 650      * @param config
 651      * @param scientificName
 652      * @param preferredFlag
 653      * @param i
 654      * @param taxonnametoinsert
 655      * @param preferredtaxonnametoinsert
 656      * @param taxonName
 657      * @return
 658      */
 659     private Taxon getTaxon(Abcd206ImportState state, String scientificName, int i, Rank rank, String nomenclature) {
 660         Abcd206ImportConfigurator config = state.getConfig();
 661
 662         if (rank == null) {
 663             System.out.println("getTaxon "+scientificName);
 664         } else {
 665             System.out.println("getTaxon "+scientificName+", "+rank.generateTitle());
 666         }
 667         Taxon taxon = null;
 668         NonViralName<?> taxonName = null;
 669
 670         if (config.isDoReUseTaxon()){
 671             List<TaxonBase> c = null;
 672             try {
 673                 Taxon cc= getTaxonService().findBestMatchingTaxon(scientificName);
 674                 if (cc != null && cc.getSec()!=null && cc.getSec().getTitleCache().equalsIgnoreCase(ref.getTitleCache())){
 675                     taxon=cc;
 676                 }
 677                 else{
 678                     c = getTaxonService().searchTaxaByName(scientificName, ref);
 679                     for (TaxonBase<?> b : c) {
 680                         taxon = (Taxon) b;
 681                     }
 682                 }
 683             } catch (Exception e) {
 684                 logger.info("Searchtaxabyname failed" + e);
 685                 taxon = null;
 686             }
 687         }
 688         if (!config.isDoReUseTaxon() || taxon == null){
 689             System.out.println("create new taxonName instance");
 690             if (config.isDoAutomaticParsing()){
 691                 taxonName = parseScientificName(scientificName);
 692             }
 693             else{
 694                 if (i>=0 && (dataHolder.atomisedIdentificationList != null || dataHolder.atomisedIdentificationList.size() > 0)) {
 695                     taxonName = setTaxonNameByType(dataHolder.atomisedIdentificationList.get(i), scientificName);
 696                 } else {
 697                     taxonName=null;
 698                 }
 699             }
 700             if(taxonName == null){
 701                 taxonName = NonViralName.NewInstance(rank);
 702                 taxonName.setFullTitleCache(scientificName,true);
 703                 taxonName.setTitleCache(scientificName, true);
 704             }
 705             System.out.println("ADD NEW TAXON *"+taxonName.getRank()+"*"+taxonName);
 706             if (rank != null && (taxonName.getRank() ==null || taxonName.getRank().toString().trim().isEmpty())) {
 707                 taxonName.setRank(rank);
 708             }
 709             getNameService().save(taxonName);
 710             taxon = Taxon.NewInstance(taxonName, ref); //sec set null
 711             getTaxonService().save(taxon);
 712             refreshTransaction(state);
 713            taxon= (Taxon) getTaxonService().find(taxon.getUuid());
 714         }
 715
 716
 717
 718         System.out.println("taxon.getUuid() suite :"+taxon.getUuid());
 719         return taxon;
 720     }
 721
 722
 723
 724         /**
 725          * getParentTaxon : get the taxonomic hierarchy for the current Taxon
 726          *
 727          * @param taxon
 728          * @param taxonName
 729          * @param originalName
 730          * @param config
 731          * @return a map with the parenttaxon and the parenttaxonname
 732          */
 733         private HashMap<Taxon, NonViralName<?>> getParentTaxon(Abcd206ImportState state, Taxon taxon, NonViralName<?> taxonName, NonViralName<?> originalName) {
 734
 735                 Abcd206ImportConfigurator config = state.getConfig();
 736                 Taxon parenttaxon = null;
 737                 NonViralName<?> parentName = null;
 738                 List<TaxonBase> c = null;
 739
 740                 List<String> highername = new ArrayList<String>();
 741                 Rank higherrank = null;
 742                 Rank taxonrank = taxonName.getRank();
 743
 744                 if(DEBUG) logger.info("getParentTaxon childname " + taxonName.getFullTitleCache() + ", rank " + taxonrank + ", originalname " + originalName.getFullTitleCache());
 745
 746                 HashMap<Taxon, NonViralName<?>> map = new HashMap<Taxon, NonViralName<?>>();
 747
 748
 749                 refreshTransaction(state);
 750
 751                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 752
 753                 if (taxonrank.isGenus()) {
 754                         // to change and add test DoReusetaxa
 755                         for (TaxonNode p : classification.getAllNodes()) {
 756                                 if(DEBUG) logger.info("p UUID "+p.getUuid().toString());
 757                                 if (classification.getTopmostNode(p.getTaxon()) == null) {
 758                                         if(DEBUG){
 759                                                 logger.info("taxon1 "+p.getTaxon().getTitleCache());
 760                                                 logger.info("taxon2 "+taxon.getTitleCache());
 761                                         }
 762                                         if (taxon.getTitleCache().contains(p.getTaxon().getTitleCache().split(SEC + ref)[0])) {
 763                                                 this.addParentChild(state, p.getTaxon(), taxon);
 764
 765                                                 refreshTransaction(state);
 766                                                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 767
 768                                                 break;
 769                                         }
 770                                 }
 771                         }
 772                         // add the genus to the root of the classification
 773                         TaxonNode p = this.addChildTaxon(state, taxon);
 774                         this.addChildNode(state, p);
 775
 776                         refreshTransaction(state);
 777
 778                         taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 779
 780                         return map;
 781                 }
 782
 783                 else if (taxonrank.isInfraGeneric()) {
 784                         if(DEBUG) logger.info("isInfrageneric");
 785                         highername.add(originalName.getGenusOrUninomial());
 786                         higherrank = Rank.GENUS();
 787                 }
 788
 789                 else if (taxonrank.isSpecies()) {
 790                         if(DEBUG) logger.info("isSpecies");
 791                         if (originalName.getGenusOrUninomial() != null) {
 792                                 highername.add(originalName.getGenusOrUninomial());
 793                                 higherrank = Rank.GENUS();
 794                         }
 795                         else {
 796                                 highername.add(originalName.getFullTitleCache().trim().split(" ")[0]);
 797                                 higherrank = Rank.GENUS();
 798                         }
 799                         if (originalName.getInfraGenericEpithet() != null) {
 800                                 highername.add(originalName.getInfraGenericEpithet());
 801                                 higherrank = Rank.INFRAGENUS();
 802                         }
 803                 }
 804
 805                 else if (taxonrank.isInfraSpecific()) {
 806                         if(DEBUG) logger.info("isInfraSpecies");
 807                         if (originalName.getGenusOrUninomial() != null){
 808                                 highername.add(originalName.getGenusOrUninomial());
 809                         }
 810                         if (originalName.getInfraGenericEpithet() != null) {
 811                                 highername.add(originalName.getInfraGenericEpithet());
 812                         }
 813                         if (originalName.getSpecificEpithet() != null){
 814                                 highername.add(originalName.getSpecificEpithet());
 815                         }
 816                         higherrank = Rank.SPECIES();
 817                 }
 818
 819                 String highernamestr = StringUtils.join(highername.iterator(), " ").split(SEC + ref.getTitleCache())[0].trim();
 820                 if(DEBUG) logger.info("higherNamest :: " + highernamestr);
 821                 if (config.isDoReUseTaxon() && highername.size() > 0 && isNotBlank(highernamestr)) {
 822                         boolean parentFound = false;
 823                         try {
 824                                 c = getTaxonService().searchTaxaByName(highernamestr, ref);
 825
 826                                 for (TaxonBase<?> b : c) {
 827                                         parenttaxon = (Taxon) b;
 828                                         Iterator<TaxonNode> it = parenttaxon.getTaxonNodes().iterator();
 829                                         // logger.warn("ICI2");
 830                                         while (it.hasNext()) {
 831                                                 TaxonNode tmpNode = it.next();
 832                                                 Taxon tmp = tmpNode.getTaxon();
 833                                                 if (tmp.getTitleCache().split(SEC + ref.getTitleCache())[0].trim().equalsIgnoreCase(highernamestr)) {
 834                                                         parenttaxon = tmp;
 835                                                         parentName = (NonViralName<?>) b.getName();
 836                                                         parentFound = true;
 837                                                 }
 838                                         }
 839                                         if(DEBUG){
 840                                                 if (!parentFound)
 841                                                         logger.info("parent not found");
 842                                                 else
 843                                                         logger.info("parent found: " + parenttaxon.getTitleCache());
 844                                         }
 845
 846                                 }
 847                         } catch (Exception e) {
 848                                 logger.info("Problem while trying to reuse existing taxon" + e);
 849                                 parenttaxon = null;
 850                         }
 851                         if (!parentFound)
 852                                 parenttaxon = null;
 853                 }
 854
 855                 if ((parenttaxon == null && highername.size() > 0 && isNotBlank(highernamestr)) || !config.isDoReUseTaxon()) {
 856                         // logger.info("ICI BIS");
 857                         parentName = NonViralName.NewInstance(null);
 858                         parentName.setFullTitleCache(highernamestr);
 859                         parentName.setNameCache(highernamestr);
 860                         parentName.setRank(higherrank);
 861                         if (higherrank == Rank.GENUS()){
 862                                 parentName.setGenusOrUninomial(highernamestr.split(" ")[0]);
 863                         }
 864
 865                         getNameService().save(parentName);
 866                         parenttaxon = Taxon.NewInstance(parentName, ref);
 867                         getTaxonService().save(parenttaxon);
 868
 869                 }
 870
 871                 map.put(parenttaxon, parentName);
 872                 return map;
 873         }
 874
 875         private TaxonNode addChildNode(Abcd206ImportState state, TaxonNode childNode){
 876                 TaxonNode re =null;
 877                 boolean exists=false;
 878                 if (state.getConfig().isDoReUseTaxon()){
 879                         Taxon taxon = childNode.getTaxon();
 880                         Set<TaxonNode> allNodes = classification.getAllNodes();
 881                         Taxon tmp;
 882                         if (allNodes.size()>0)
 883                                 for (TaxonNode tn:allNodes){
 884                                         tmp = tn.getTaxon();
 885                                         if (tmp.equals(taxon)){
 886                                                 re= tn;
 887                                                 exists=true;
 888                                                 break;
 889                                         }
 890                                 }
 891                 }
 892                 if(! state.getConfig().isDoReUseTaxon() || !exists){
 893                         refreshTransaction(state);
 894                         childNode = getTaxonNodeService().find(childNode.getUuid());
 895                         re= classification.addChildNode(childNode, ref, "", null);
 896                 }
 897                 return re;
 898         }
 899
 900         private void addParentChild(Abcd206ImportState state, Taxon parent, Taxon child){
 901                 if(DEBUG) logger.info("addParentChild");
 902                 boolean exists = false;
 903                 Taxon taxonFromHiber = null;
 904                 if (state.getConfig().isDoReUseTaxon()){
 905                         Set<TaxonNode> allNodes = classification.getAllNodes();
 906                         Taxon tmp;
 907                         if (allNodes.size()>0){
 908                                 for (TaxonNode tn:allNodes){
 909                                         tmp = tn.getTaxon();
 910                                         if (tmp.equals(parent)){
 911                                                 taxonFromHiber = tmp;
 912                                         }
 913                                 }
 914                         }
 915                         if (taxonFromHiber != null){
 916                                 Set<TaxonNode> children = taxonFromHiber.getTaxonNodes();
 917                                 if (children.size()>0)
 918                                         for (TaxonNode cn:children){
 919                                                 Taxon tmpt = cn.getTaxon();
 920                                                 if (tmpt.equals(child))
 921                                                         exists=true;
 922                                         }
 923                         }
 924                 }
 925                 if (! state.getConfig().isDoReUseTaxon() || !exists){
 926                         refreshTransaction(state);
 927                         parent = (Taxon) getTaxonService().find(parent.getUuid());
 928                         child = (Taxon) getTaxonService().find(child.getUuid());
 929                         classification.addParentChild(parent,child, ref, null);
 930                 }
 931         }
 932
 933         private TaxonNode addChildTaxon(Abcd206ImportState state,Taxon child){
 934                 TaxonNode re =null;
 935                 boolean exists=false;
 936                 if (state.getConfig().isDoReUseTaxon()){
 937                         Set<TaxonNode> allNodes = classification.getAllNodes();
 938                         Taxon tmp;
 939                         if (allNodes.size()>0){
 940                                 for (TaxonNode tn:allNodes){
 941                                         tmp = tn.getTaxon();
 942                                         if (tmp.equals(child)){
 943                                                 re = tn;
 944                                                 exists=true;
 945                                                 break;
 946                                         }
 947                                 }
 948                         }
 949                 }
 950                 if(! state.getConfig().isDoReUseTaxon() || !exists){
 951                         refreshTransaction(state);
 952                         child = (Taxon) getTaxonService().find(child.getUuid());
 953                         re= classification.addChildTaxon(child, ref, "", null);
 954                 }
 955                 return re;
 956         }
 957
 958
 959     /**
 960      * @param state
 961      * @param scientificName
 962      * @param parseInt
 963      * @return
 964      */
 965     private Taxon getTaxon(Abcd206ImportState state, String scientificName, int index, String nomenclature) {
 966         return getTaxon(state, scientificName, index, null, nomenclature);
 967     }
 968
 969     /**
 970      * HandleIdentifications : get the scientific names present in the ABCD
 971      * document and store link them with the observation/specimen data
 972      *
 973      * @param config
 974      */
 975     private void handleIdentifications(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
 976
 977         Abcd206ImportConfigurator config = state.getConfig();
 978
 979         String fullScientificNameString;
 980         Taxon taxon = null;
 981         Rank.GENUS();
 982         Rank.FAMILY();
 983
 984         String scientificName = "";
 985         boolean preferredFlag = false;
 986         boolean onePreferred = false;
 987
 988         List<String> scientificNames = new ArrayList<String>();
 989         if (dataHolder.nomenclatureCode == ""){
 990             dataHolder.nomenclatureCode = config.getNomenclaturalCode().toString();
 991         }
 992
 993         for (int i = 0; i < dataHolder.identificationList.size(); i++) {
 994
 995             fullScientificNameString = dataHolder.identificationList.get(i);
 996             fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
 997
 998             if (fullScientificNameString.indexOf(PREFERRED) != -1) {
 999                 scientificName = fullScientificNameString.split(PREFERRED)[0];
1000                 String pTmp = fullScientificNameString.split(PREFERRED)[1].split(CODE)[0];
1001                 if (pTmp.equals("1") || pTmp.toLowerCase().indexOf("true") != -1) {
1002                     preferredFlag = true;
1003                     onePreferred = true;
1004                 }
1005                 else {
1006                     preferredFlag = false;
1007                 }
1008             }
1009             else {
1010                 scientificName = fullScientificNameString;
1011             }
1012             if(DEBUG) {
1013                 logger.info("fullscientificname " + fullScientificNameString + ", *" + dataHolder.nomenclatureCode + "*");
1014             }
1015             if (fullScientificNameString.indexOf(CODE) != -1) {
1016                 if (fullScientificNameString.indexOf(':') != -1) {
1017                     dataHolder.nomenclatureCode = fullScientificNameString.split(CODE)[1].split(COLON)[1];
1018                 }
1019                 else{
1020                     dataHolder.nomenclatureCode = fullScientificNameString.split(CODE)[1];
1021                 }
1022             }
1023             scientificNames.add(scientificName+SPLITTER+preferredFlag+SPLITTER+i);
1024         }
1025         for (String name:scientificNames) {
1026             scientificName = name.split(SPLITTER)[0];
1027             String pref = name.split(SPLITTER)[1];
1028             String index = name.split(SPLITTER)[2];
1029             if (pref.equalsIgnoreCase("true") || scientificNames.size()==1) {
1030                 preferredFlag = true;
1031             } else {
1032                 preferredFlag =false;
1033             }
1034             taxon = getTaxon(state, scientificName,Integer.parseInt(index),dataHolder.nomenclatureCode);
1035             addTaxonNode(taxon, state,dataHolder.nomenclatureCode);
1036             taxon = (Taxon) getTaxonService().find(taxon.getUuid());
1037             linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
1038             //            refreshTransaction();
1039         }
1040         refreshTransaction(state);
1041         taxon = (Taxon) getTaxonService().find(taxon.getUuid());
1042     }
1043
1044     /**
1045      * @param taxon
1046      * @param config
1047      * @return
1048      */
1049     private void addTaxonNode(Taxon taxon, Abcd206ImportState state, String nomenclature) {
1050         logger.info("link taxon to a taxonNode");
1051         boolean exist = false;
1052         for (TaxonNode p : classification.getAllNodes()){
1053             if(p.getTaxon().equals(taxon)) {
1054                 exist =true;
1055             }
1056         }
1057         if (!exist){
1058             addParentTaxon(taxon, state,nomenclature);
1059         }
1060         refreshTransaction(state);
1061     }
1062
1063     private void addParentTaxon(Taxon taxon, Abcd206ImportState state,String nomenclature){
1064         System.out.println("addParentTaxon "+taxon.getTitleCache());
1065
1066                 NonViralName<?>  nvname = CdmBase.deproxy(taxon.getName(), NonViralName.class);
1067             Rank rank = nvname.getRank();
1068             Taxon genus =null;
1069             Taxon subgenus =null;
1070             Taxon species = null;
1071             Taxon subspecies = null;
1072             if (rank.isLower(Rank.GENUS() )){
1073                 String prefix = nvname.getGenusOrUninomial();
1074                 genus = getTaxon(state, prefix, -1, Rank.GENUS(),nomenclature);
1075                 saveOrUpdateClassification(null, genus);
1076             }
1077             if (rank.isLower(Rank.SUBGENUS())){
1078                 String prefix = nvname.getGenusOrUninomial();
1079                 String name = nvname.getInfraGenericEpithet();
1080                 if (name != null){
1081                     subgenus = getTaxon(state, prefix+" "+name, -1, Rank.SUBGENUS(),nomenclature);
1082                     saveOrUpdateClassification(genus, subgenus);
1083                 }
1084             }
1085             if (rank.isLower(Rank.SPECIES())){
1086                 if (subgenus!=null){
1087                     String prefix = nvname.getGenusOrUninomial();
1088                     String name = nvname.getInfraGenericEpithet();
1089                     String spe = nvname.getSpecificEpithet();
1090                     if (spe != null){
1091                         species = getTaxon(state, prefix+" "+name+" "+spe, -1, Rank.SPECIES(),nomenclature);
1092                         saveOrUpdateClassification(subgenus, species);
1093                     }
1094                 }
1095                 else{
1096                     String prefix = nvname.getGenusOrUninomial();
1097                     String name = nvname.getSpecificEpithet();
1098                     if (name != null){
1099                         species = getTaxon(state, prefix+" "+name, -1, Rank.SPECIES(),nomenclature);
1100                         saveOrUpdateClassification(genus, species);
1101                     }
1102                 }
1103             }
1104             if (rank.isInfraSpecific()){
1105                     subspecies = getTaxon(state, nvname.getFullTitleCache(), -1, Rank.SUBSPECIES(),nomenclature);
1106                     saveOrUpdateClassification(species, subspecies);
1107             }
1108         }
1109
1110     /**
1111      * @param currentTaxon
1112      * @param taxon
1113      */
1114     private void saveOrUpdateClassification(Taxon parent, Taxon child) {
1115         System.out.println("ADD CLASSIFICATION parent child "+parent+"," +child);
1116         if (parent != null) {
1117             parent = (Taxon) getTaxonService().find(parent.getUuid());
1118             child = (Taxon) getTaxonService().find(child.getUuid());
1119             classification.addParentChild(parent, child, ref, "");
1120         }
1121         if (parent == null) {
1122             child = (Taxon) getTaxonService().find(child.getUuid());
1123             classification.addChildTaxon(child, ref, "", null);
1124         }
1125         getClassificationService().saveOrUpdate(classification);
1126     }
1127
1128
1129
1130         private NonViralName<?> parseScientificName(String scientificName) {
1131                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1132                 NonViralName<?> taxonName = null;
1133                 boolean problem = false;
1134
1135                 if(DEBUG){
1136                         logger.info("parseScientificName " + dataHolder.nomenclatureCode.toString());
1137                 }
1138
1139                 if (dataHolder.nomenclatureCode.toString().equals("Zoological") || dataHolder.nomenclatureCode.toString().contains("ICZN")) {
1140                         taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
1141                         if (taxonName.hasProblem()) {
1142                                 problem = true;
1143                         }
1144                 }
1145                 if (dataHolder.nomenclatureCode.toString().equals("Botanical") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1146                         taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
1147                         if (taxonName.hasProblem()) {
1148                                 problem = true;
1149                         }
1150                 }
1151                 if (dataHolder.nomenclatureCode.toString().equals("Bacterial") || dataHolder.nomenclatureCode.toString().contains("ICBN")) {
1152                         taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
1153                         if (taxonName.hasProblem()) {
1154                                 problem = true;
1155                         }
1156                 }
1157                 if (dataHolder.nomenclatureCode.toString().equals("Cultivar") || dataHolder.nomenclatureCode.toString().contains("ICNCP")) {
1158                         taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
1159                         if (taxonName.hasProblem()) {
1160                                 problem = true;
1161                         }
1162                 }
1163                 if (problem) {
1164                         logger.info("Parsing with problem in parseScientificName " + scientificName);
1165                         return null;
1166                 }
1167                 return taxonName;
1168
1169         }
1170
1171         private NonViralName<?> setTaxonNameByType(
1172                         HashMap<String, String> atomisedMap, String fullName) {
1173                 boolean problem = false;
1174                 if(DEBUG) logger.info("settaxonnamebytype " + dataHolder.nomenclatureCode.toString());
1175
1176                 if (dataHolder.nomenclatureCode.equals("Zoological")) {
1177                         NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
1178                         taxonName.setFullTitleCache(fullName, true);
1179                         taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1180                         taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1181                         taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
1182                         taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
1183
1184                         if (taxonName.getGenusOrUninomial() != null){
1185                                 taxonName.setRank(Rank.GENUS());
1186                         }
1187
1188                         else if (taxonName.getInfraGenericEpithet() != null){
1189                                 taxonName.setRank(Rank.SUBGENUS());
1190                         }
1191
1192                         else if (taxonName.getSpecificEpithet() != null){
1193                                 taxonName.setRank(Rank.SPECIES());
1194                         }
1195
1196                         else if (taxonName.getInfraSpecificEpithet() != null){
1197                                 taxonName.setRank(Rank.SUBSPECIES());
1198                         }
1199
1200                         Team team = null;
1201                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1202                                 team = Team.NewInstance();
1203                                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1204                         }
1205                         else {
1206                                 if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1207                                         team = Team.NewInstance();
1208                                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1209                                 }
1210                         }
1211                         if (team != null) {
1212                                 taxonName.setBasionymAuthorTeam(team);
1213                         }
1214                         else {
1215                                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1216                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1217                                 }
1218                                 else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1219                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
1220                                 }
1221                         }
1222                         if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1223                                 team = Team.NewInstance();
1224                                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1225                                 taxonName.setCombinationAuthorTeam(team);
1226                         }
1227                         if (taxonName.hasProblem()) {
1228                                 logger.info("pb ICZN");
1229                                 problem = true;
1230                         }
1231                         else {
1232                                 return taxonName;
1233                         }
1234                 }
1235                 else if (dataHolder.nomenclatureCode.equals("Botanical")) {
1236                         BotanicalName taxonName = (BotanicalName) parseScientificName(fullName);
1237                         if (taxonName != null){
1238                                 return taxonName;
1239                         }
1240                         else{
1241                                 taxonName = BotanicalName.NewInstance(null);
1242                         }
1243                         taxonName.setFullTitleCache(fullName, true);
1244                         taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1245                         taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
1246                         taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
1247                         try {
1248                                 taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
1249                         } catch (Exception e) {
1250                                 if (taxonName.getGenusOrUninomial() != null){
1251                                         taxonName.setRank(Rank.GENUS());
1252                                 }
1253                                 else if (taxonName.getInfraGenericEpithet() != null){
1254                                         taxonName.setRank(Rank.SUBGENUS());
1255                                 }
1256                                 else if (taxonName.getSpecificEpithet() != null){
1257                                         taxonName.setRank(Rank.SPECIES());
1258                                 }
1259                                 else if (taxonName.getInfraSpecificEpithet() != null){
1260                                         taxonName.setRank(Rank.SUBSPECIES());
1261                                 }
1262                         }
1263                         Team team = null;
1264                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1265                                 team = Team.NewInstance();
1266                                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1267                                 taxonName.setBasionymAuthorTeam(team);
1268                         }
1269                         if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1270                                 team = Team.NewInstance();
1271                                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
1272                                 taxonName.setCombinationAuthorTeam(team);
1273                         }
1274                         if (team == null) {
1275                                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1276                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1277                                 }
1278                                 else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1279                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
1280                                 }
1281                         }
1282                         if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1283                                 team = Team.NewInstance();
1284                                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1285                                 taxonName.setCombinationAuthorTeam(team);
1286                         }
1287                         if (taxonName.hasProblem()) {
1288                                 logger.info("pb ICBN");
1289                                 problem = true;
1290                         }
1291                         else {
1292                                 return taxonName;
1293                         }
1294                 }
1295                 else if (dataHolder.nomenclatureCode.equals("Bacterial")) {
1296                         NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
1297                         taxonName.setFullTitleCache(fullName, true);
1298                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
1299                         taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1300                         taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
1301                         taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
1302
1303                         if (taxonName.getGenusOrUninomial() != null){
1304                                 taxonName.setRank(Rank.GENUS());
1305                         }
1306                         else if (taxonName.getInfraGenericEpithet() != null){
1307                                 taxonName.setRank(Rank.SUBGENUS());
1308                         }
1309                         else if (taxonName.getSpecificEpithet() != null){
1310                                 taxonName.setRank(Rank.SPECIES());
1311                         }
1312                         else if (taxonName.getInfraSpecificEpithet() != null){
1313                                 taxonName.setRank(Rank.SUBSPECIES());
1314                         }
1315
1316                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1317                                 Team team = Team.NewInstance();
1318                                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1319                                 taxonName.setCombinationAuthorTeam(team);
1320                         }
1321                         if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
1322                                 Team team = Team.NewInstance();
1323                                 team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
1324                                 taxonName.setBasionymAuthorTeam(team);
1325                         }
1326                         if (taxonName.hasProblem()) {
1327                                 logger.info("pb ICNB");
1328                                 problem = true;
1329                         }
1330                         else {
1331                                 return taxonName;
1332                         }
1333                 }
1334                 else if (dataHolder.nomenclatureCode.equals("Cultivar")) {
1335                         CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
1336
1337                         if (taxonName.hasProblem()) {
1338                                 logger.info("pb ICNCP");
1339                                 problem = true;
1340                         }
1341                         else {
1342                                 return taxonName;
1343                         }
1344                         return taxonName;
1345                 }
1346
1347                 if (problem) {
1348                         logger.info("Problem im setTaxonNameByType ");
1349                         NonViralName<?> taxonName = NonViralName.NewInstance(null);
1350                         taxonName.setFullTitleCache(fullName, true);
1351                         return taxonName;
1352                 }
1353                 NonViralName<?> tn = NonViralName.NewInstance(null);
1354                 return tn;
1355         }
1356
1357         private String getFromMap(HashMap<String, String> atomisedMap, String key) {
1358                 String value = null;
1359                 if (atomisedMap.containsKey(key)) {
1360                         value = atomisedMap.get(key);
1361                 }
1362
1363                 try {
1364                         if (value != null && key.matches(".*Year.*")) {
1365                                 value = value.trim();
1366                                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
1367                                         String tmp = value.split("[0-9]{4}$")[0];
1368                                         int year = Integer.parseInt(value.split(tmp)[1]);
1369                                         if (year >= 1752) {
1370                                                 value = tmp;
1371                                         }
1372                                         else {
1373                                                 value = null;
1374                                         }
1375                                 }
1376                                 else {
1377                                         value = null;
1378                                 }
1379                         }
1380                 }
1381                 catch (Exception e) {
1382                         value = null;
1383                 }
1384                 return value;
1385         }
1386
1387         private void compareABCDtoCDM(URI urlFileName, List<String> knownElts, Abcd206XMLFieldGetter abcdFieldGetter) {
1388                 try {
1389                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
1390                         DocumentBuilder constructeur = factory.newDocumentBuilder();
1391                         URL url = urlFileName.toURL();
1392                         Object o = url.getContent();
1393                         InputStream is = (InputStream) o;
1394                         Document document = constructeur.parse(is);
1395                         Element root = document.getDocumentElement();
1396                         abcdFieldGetter.traverse(root);
1397                 }
1398                 catch (ParserConfigurationException e){
1399                         e.printStackTrace();
1400                 }
1401                 catch (SAXException e) {
1402                         e.printStackTrace();
1403                 }
1404                 catch (IOException e) {
1405                         e.printStackTrace();
1406                 }
1407                 Set<String> elts = dataHolder.allABCDelements.keySet();
1408                 Iterator<String> it = elts.iterator();
1409                 String elt;
1410                 while (it.hasNext()) {
1411                         elt = it.next();
1412                         if (knownElts.indexOf(elt) == -1) {
1413                                 if(DEBUG) logger.info("Unmerged ABCD element: " + elt + " - "+ dataHolder.allABCDelements.get(elt));
1414                         }
1415                 }
1416         }
1417
1418     /**
1419      * @param unitsList
1420      * @param state
1421      */
1422     private void prepareCollectors(Abcd206ImportState state, NodeList unitsList, Abcd206XMLFieldGetter abcdFieldGetter) {
1423         List<String> collectors = new ArrayList<String>();
1424         List<String> teams = new ArrayList<String>();
1425         List<List<String>> collectorinteams = new ArrayList<List<String>>();
1426
1427         for (int i = 0; i < unitsList.getLength(); i++) {
1428             this.getCollectorsFromXML((Element) unitsList.item(i), abcdFieldGetter);
1429             for (String agent : dataHolder.gatheringAgentList) {
1430                 collectors.add(agent);
1431             }
1432             List<String> tmpTeam = new ArrayList<String>(new HashSet<String>(dataHolder.gatheringTeamList));
1433             if(!tmpTeam.isEmpty()) {
1434                 teams.add(StringUtils.join(tmpTeam.toArray()," & "));
1435             }
1436             for (String agent:tmpTeam) {
1437                 collectors.add(agent);
1438             }
1439         }
1440
1441         List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
1442         List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
1443
1444
1445         //existing teams in DB
1446         Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
1447         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
1448
1449         Set<UUID> uuids = new HashSet<UUID>();
1450         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
1451             uuids.add(hibernateT.getUuid());
1452         }
1453         if (!uuids.isEmpty()){
1454             List<AgentBase> existingTeams = getAgentService().find(uuids);
1455             for (AgentBase<?> existingP:existingTeams){
1456                 titleCacheTeam.put(existingP.getTitleCache(),(Team) existingP);
1457             }
1458         }
1459
1460
1461         Map<String,UUID> teamMap = new HashMap<String, UUID>();
1462         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
1463             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
1464         }
1465
1466         //existing persons in DB
1467         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
1468         Map<String,Person> titleCachePerson = new HashMap<String, Person>();
1469         uuids = new HashSet<UUID>();
1470         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
1471             uuids.add(hibernateP.getUuid());
1472         }
1473
1474         if (!uuids.isEmpty()){
1475             List<AgentBase> existingPersons = getAgentService().find(uuids);
1476             for (AgentBase<?> existingP:existingPersons){
1477                 titleCachePerson.put(existingP.getTitleCache(),(Person) existingP);
1478             }
1479         }
1480
1481         Map<String,UUID> personMap = new HashMap<String, UUID>();
1482         for (UuidAndTitleCache<Person> person:hiberPersons){
1483             personMap.put(person.getTitleCache(), person.getUuid());
1484         }
1485
1486         java.util.Collection<AgentBase> personToadd = new ArrayList<AgentBase>();
1487         java.util.Collection<AgentBase> teamToAdd = new ArrayList<AgentBase>();
1488
1489         for (String collector:collectorsU){
1490             Person p = Person.NewInstance();
1491             p.setTitleCache(collector,true);
1492             if (!personMap.containsKey(p.getTitleCache())){
1493                 personToadd.add(p);
1494             }
1495         }
1496         for (String team:teamsU){
1497             Team p = Team.NewInstance();
1498             p.setTitleCache(team,true);
1499             if (!teamMap.containsKey(p.getTitleCache())){
1500                 teamToAdd.add(p);
1501             }
1502         }
1503
1504         if(!personToadd.isEmpty()){
1505             Map<UUID, AgentBase> uuuidPerson = getAgentService().save(personToadd);
1506             for (UUID u:uuuidPerson.keySet()){
1507                 titleCachePerson.put(uuuidPerson.get(u).getTitleCache(),(Person) uuuidPerson.get(u) );
1508             }
1509         }
1510
1511         Person ptmp ;
1512         Map <String,Integer>teamdone = new HashMap<String, Integer>();
1513         for (List<String> collteam: collectorinteams){
1514             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
1515                 Team team = new Team();
1516                 boolean em =true;
1517                 for (String collector:collteam){
1518                     ptmp = Person.NewInstance();
1519                     ptmp.setTitleCache(collector,true);
1520                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
1521                     team.addTeamMember(p2);
1522                     em=false;
1523                 }
1524                 if (!em) {
1525                     teamToAdd.add(team);
1526                 }
1527                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
1528             }
1529         }
1530
1531         if(!teamToAdd.isEmpty()){
1532             Map<UUID, AgentBase> uuuidTeam =  getAgentService().save(teamToAdd);
1533             for (UUID u:uuuidTeam.keySet()){
1534                 titleCacheTeam.put(uuuidTeam.get(u).getTitleCache(), (Team) uuuidTeam.get(u) );
1535             }
1536         }
1537
1538         state.getConfig().setTeams(titleCacheTeam);
1539         state.getConfig().setPersons(titleCachePerson);
1540     }
1541
1542         @Override
1543         protected boolean isIgnore(Abcd206ImportState state) {
1544                 return false;
1545         }
1546
1547
1548 }