cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.net.URI;
  15 import java.net.URL;
  16 import java.util.ArrayList;
  17 import java.util.HashMap;
  18 import java.util.Iterator;
  19 import java.util.List;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import javax.xml.parsers.DocumentBuilder;
  24 import javax.xml.parsers.DocumentBuilderFactory;
  25 import javax.xml.parsers.ParserConfigurationException;
  26
  27 import org.apache.log4j.Logger;
  28 import org.springframework.stereotype.Component;
  29 import org.springframework.transaction.TransactionStatus;
  30 import org.w3c.dom.Document;
  31 import org.w3c.dom.Element;
  32 import org.w3c.dom.Node;
  33 import org.w3c.dom.NodeList;
  34 import org.xml.sax.SAXException;
  35
  36 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  37 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  38 import eu.etaxonomy.cdm.io.common.ICdmIO;
  39 import eu.etaxonomy.cdm.io.specimen.SpecimenImportBase;
  40 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  41 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  42 import eu.etaxonomy.cdm.model.agent.Institution;
  43 import eu.etaxonomy.cdm.model.agent.Team;
  44 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  45 import eu.etaxonomy.cdm.model.description.Feature;
  46 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  47 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  48 import eu.etaxonomy.cdm.model.location.NamedArea;
  49 import eu.etaxonomy.cdm.model.media.Media;
  50 import eu.etaxonomy.cdm.model.name.BacterialName;
  51 import eu.etaxonomy.cdm.model.name.BotanicalName;
  52 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  53 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  54 import eu.etaxonomy.cdm.model.name.NonViralName;
  55 import eu.etaxonomy.cdm.model.name.Rank;
  56 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  57 import eu.etaxonomy.cdm.model.occurrence.Collection;
  58 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  59 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceBase;
  61 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  62 import eu.etaxonomy.cdm.model.taxon.Taxon;
  63 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  64 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  65
  66
  67 /**
  68  * @author p.kelbert
  69  * @created 20.10.2008
  70  * @version 1.0
  71  */
  72 @Component
  73 public class Abcd206Import extends SpecimenImportBase<Abcd206ImportConfigurator, Abcd206ImportState> implements ICdmIO<Abcd206ImportState> {
  74         private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  75
  76
  77         public Abcd206Import() {
  78                 super();
  79         }
  80
  81
  82         @Override
  83         protected boolean doCheck(Abcd206ImportState state) {
  84                 logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
  85                 return true;
  86         }
  87
  88
  89         @Override
  90         public boolean doInvoke(Abcd206ImportState state){
  91                 logger.info("INVOKE Specimen Import from ABCD2.06 XML File");
  92                 boolean result = true;
  93                 //AbcdIO test = new AbcdIO();
  94                 URI sourceName = state.getConfig().getSource();
  95                 NodeList unitsList = getUnitsNodeList(sourceName);
  96                 if (unitsList != null){
  97                         String message = "nb units to insert: "+unitsList.getLength();
  98                         logger.info(message);
  99                         updateProgress(state, message);
 100
 101                         Abcd206DataHolder dataHolder = new Abcd206DataHolder();
 102
 103                         for (int i=0 ; i<unitsList.getLength() ; i++){
 104                                 this.setUnitPropertiesXML((Element)unitsList.item(i), dataHolder);
 105                                 result &= this.handleSingleUnit(state, dataHolder);
 106
 107                                 //compare the ABCD elements added in to the CDM and the unhandled ABCD elements
 108                                 compareABCDtoCDM(sourceName, dataHolder.knownABCDelements, dataHolder);
 109
 110                                 //reset the ABCD elements added in CDM
 111                                 //knownABCDelements = new ArrayList<String>();
 112                                 dataHolder.allABCDelements = new HashMap<String,String>();
 113                         }
 114                 }
 115
 116                 return result;
 117
 118         }
 119
 120         /*
 121          * Store the unit with its Gathering informations in the CDM
 122          */
 123         private boolean handleSingleUnit(Abcd206ImportState state, Abcd206DataHolder dataHolder){
 124                 boolean result = true;
 125
 126                 Abcd206ImportConfigurator config = state.getConfig();
 127
 128                 TransactionStatus tx = startTransaction();
 129                 try {
 130                         updateProgress(state, "Importing data for unit: " + dataHolder.unitID);
 131
 132 //                      ReferenceBase sec = Database.NewInstance();
 133 //                      sec.setTitleCache("XML DATA");
 134                         ReferenceBase sec = config.getTaxonReference();
 135
 136                         //create facade
 137                         DerivedUnitFacade derivedUnitFacade = getFacade(dataHolder);
 138
 139
 140                         //handle identifications
 141                         handleIdentifications(config, derivedUnitFacade, sec, dataHolder);
 142
 143                         //handle collection data
 144                         setCollectionData(config, dataHolder, derivedUnitFacade);
 145
 146                         /**
 147                          * GATHERING EVENT
 148                          */
 149
 150                         //gathering event
 151                         UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(
 152                                         getTermService(), dataHolder.locality, dataHolder.languageIso, dataHolder.longitude,
 153                                         dataHolder.latitude, dataHolder.gatheringAgentList);
 154
 155                         //country
 156                         UnitsGatheringArea unitsGatheringArea =
 157                                 new UnitsGatheringArea(dataHolder.isocountry, dataHolder.country, getOccurrenceService());
 158                         NamedArea areaCountry = unitsGatheringArea.getArea();
 159
 160                         //other areas
 161                         unitsGatheringArea = new UnitsGatheringArea(dataHolder.namedAreaList);
 162                         ArrayList<NamedArea> nas = unitsGatheringArea.getAreas();
 163                         for (NamedArea namedArea : nas){
 164                                 unitsGatheringEvent.addArea(namedArea);
 165                         }
 166
 167                         //copy gathering event to facade
 168                         GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 169                         derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 170                         derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 171                         derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 172                         derivedUnitFacade.setCountry(areaCountry);
 173                         derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 174
 175                         //TODO exsiccatum
 176
 177
 178                         //add fieldNumber
 179                         derivedUnitFacade.setFieldNumber(dataHolder.fieldNumber);
 180
 181                         //join gatheringEvent to fieldObservation
 182
 183 //                      //add Multimedia URLs
 184                         if(dataHolder.multimediaObjects.size() > 0){
 185                                 for (String multimediaObject : dataHolder.multimediaObjects){
 186                                         Media media = getImageMedia(multimediaObject, true);
 187                                         derivedUnitFacade.addDerivedUnitMedia(media);
 188                                 }
 189                         }
 190
 191                         /**
 192                          * SAVE AND STORE DATA
 193                          */
 194                         getTermService().save(areaCountry);//TODO save area sooner
 195                         for (NamedArea area : nas){
 196                                 getTermService().save(area);//save it sooner (foreach area)
 197                         }
 198                         getTermService().saveLanguageData(unitsGatheringEvent.getLocality());//TODO needs to be saved ?? save it sooner
 199
 200                         getOccurrenceService().save(derivedUnitFacade.getDerivedUnit());
 201                         logger.info("saved ABCD specimen ...");
 202
 203
 204                 } catch (Exception e) {
 205                         logger.warn("Error when reading record!!");
 206                         e.printStackTrace();
 207                         result = false;
 208                 }
 209                 commitTransaction(tx);
 210
 211                 return result;
 212         }
 213
 214         private void setCollectionData(Abcd206ImportConfigurator config,
 215                         Abcd206DataHolder dataHolder, DerivedUnitFacade derivedUnitFacade) {
 216                 //set catalogue number (unitID)
 217                 derivedUnitFacade.setCatalogNumber(dataHolder.unitID);
 218                 derivedUnitFacade.setAccessionNumber(dataHolder.accessionNumber);
 219                 derivedUnitFacade.setCollectorsNumber(dataHolder.collectorsNumber);
 220
 221
 222                 /**
 223                  * INSTITUTION & COLLECTION
 224                  */
 225                 //manage institution
 226                 Institution institution = this.getInstitution(dataHolder.institutionCode, config, dataHolder);
 227                 //manage collection
 228                 Collection collection = this.getCollection(dataHolder.collectionCode, institution, config, dataHolder);
 229                 //link specimen & collection
 230                 derivedUnitFacade.setCollection(collection);
 231         }
 232
 233
 234         private DerivedUnitFacade getFacade(Abcd206DataHolder dataHolder) {
 235                 /**
 236                  * SPECIMEN OR OBSERVATION OR LIVING
 237                  */
 238 //                      DerivedUnitBase derivedThing = null;
 239                 DerivedUnitType type = null;
 240
 241                 //create specimen
 242                 if (dataHolder.recordBasis != null){
 243                         if (dataHolder.recordBasis.toLowerCase().startsWith("s")) {//specimen
 244                                 type = DerivedUnitType.Specimen;
 245                         }else if (dataHolder.recordBasis.toLowerCase().startsWith("o")) {//observation
 246                                 type = DerivedUnitType.Observation;
 247                         }else if (dataHolder.recordBasis.toLowerCase().startsWith("l")) {//living -> fossil, herbarium sheet....???
 248                                 type = DerivedUnitType.LivingBeing;
 249                         }
 250                         if (type == null){
 251                                 logger.info("The basis of record does not seem to be known: " + dataHolder.recordBasis);
 252                                 type = DerivedUnitType.DerivedUnit;
 253                         }
 254                         //TODO fossils?
 255                 }else{
 256                         logger.info("The basis of record is null");
 257                         type = DerivedUnitType.DerivedUnit;
 258                 }
 259                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 260                 return derivedUnitFacade;
 261         }
 262
 263
 264         /*
 265          * Return the list of root nodes for an ABCD 2.06 XML file
 266          * @param fileName: the file's location
 267          * @return the list of root nodes ("Unit")
 268          */
 269         private static NodeList getUnitsNodeList(URI urlFileName){
 270                 NodeList unitList = null;
 271                 try {
 272                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 273                         DocumentBuilder builder = factory.newDocumentBuilder();
 274                         URL url = urlFileName.toURL();
 275                         Object o = url.getContent();
 276                         InputStream is = (InputStream)o;
 277                         Document document = builder.parse(is);
 278                         Element root = document.getDocumentElement();
 279                         unitList = root.getElementsByTagName("Unit");
 280
 281                 }catch(Exception e){
 282                         logger.warn(e);
 283                 }
 284                 return unitList;
 285         }
 286
 287
 288         /*
 289          * Store the unit's properties into variables
 290          * Look which unit is the preferred one
 291          * Look what kind of name it is supposed to be, for the parsing (Botanical, Zoological)
 292          * @param racine: the root node for a single unit
 293          */
 294         private void setUnitPropertiesXML(Element root, Abcd206DataHolder dataHolder){
 295                 try{
 296                         NodeList group;
 297
 298 //                      try{afficherInfos(racine, 0);}catch (Exception e) {logger.info(e);}
 299                         group = root.getChildNodes();
 300 //                      logger.info("ABCD ELEMENT not stored: "+group.item(i).getNodeName().toString()+" - value: "+group.item(i).getTextContent());
 301                         for (int i = 0; i < group.getLength(); i++){
 302                                 if (group.item(i).getNodeName().equals("Identifications")){
 303                                         group = group.item(i).getChildNodes();
 304                                         break;
 305                                 }
 306                         }
 307                         dataHolder.identificationList = new ArrayList<String>();
 308                         dataHolder.atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
 309                         dataHolder.referenceList = new ArrayList<String>();
 310                         dataHolder.multimediaObjects = new ArrayList<String>();
 311
 312                         this.getScientificNames(group, dataHolder);
 313
 314 //                      logger.info("this.identificationList "+this.identificationList.toString());
 315                         this.getIDs(root, dataHolder);
 316                         this.getRecordBasis(root, dataHolder);
 317                         this.getMultimedia(root, dataHolder);
 318                         this.getNumbers(root, dataHolder);
 319                         this.getGeolocation(root, dataHolder);
 320                         this.getGatheringPeople(root, dataHolder);
 321
 322                 } catch (Exception e) {
 323                         logger.info("Error occured while parsing XML file"+e);
 324                 }
 325         }
 326
 327         String path= "";
 328         private void getHierarchie(Node node){
 329                 while (node != null && node.getNodeName() != "DataSets"){
 330 //                      logger.info(node.getParentNode().getNodeName());
 331                         path = node.getParentNode().getNodeName()+"/"+path;
 332                         node = node.getParentNode();
 333                 }
 334 //              logger.info("path: "+path);
 335         }
 336
 337         private void getScientificNames(NodeList group, Abcd206DataHolder dataHolder){
 338                 NodeList identifications,results;
 339                 String tmpName = null;
 340                 for (int j=0; j< group.getLength(); j++){
 341                         if(group.item(j).getNodeName().equals("Identification")){
 342                                 dataHolder.nomenclatureCode ="";
 343                                 identifications = group.item(j).getChildNodes();
 344                                 for (int m=0; m<identifications.getLength();m++){
 345                                         if(identifications.item(m).getNodeName().equals("Result")){
 346                                                 results = identifications.item(m).getChildNodes();
 347                                                 for(int k=0; k<results.getLength();k++){
 348                                                         if (results.item(k).getNodeName().equals("TaxonIdentified")){
 349                                                                 tmpName=this.getScientificName(results.item(k), dataHolder);
 350                                                         }
 351                                                 }
 352                                         }else if(identifications.item(m).getNodeName().equals("PreferredFlag")){
 353                                                 if (dataHolder.nomenclatureCode != null && dataHolder.nomenclatureCode !=""){
 354                                                         dataHolder.identificationList.add(tmpName+"_preferred_"+identifications.item(m).getTextContent()+"_code_" + dataHolder.nomenclatureCode);
 355                                                 }else{
 356                                                         dataHolder.identificationList.add(tmpName+"_preferred_"+identifications.item(m).getTextContent());
 357                                                 }
 358                                                 path=identifications.item(m).getNodeName();
 359                                                 getHierarchie(identifications.item(m));
 360                                                 dataHolder.knownABCDelements.add(path);
 361                                                 path="";
 362                                         }else if (identifications.item(m).getNodeName().equals("References")){
 363                                                 this.getReferences(identifications.item(m), dataHolder);
 364                                         }
 365                                 }
 366                         }
 367                 }
 368                 boolean hasPref=false;
 369                 for (int j=0; j< group.getLength(); j++){
 370                         if(group.item(j).getNodeName().equals("Identification")){
 371                                 dataHolder.nomenclatureCode ="";
 372                                 identifications = group.item(j).getChildNodes();
 373                                 for (int m=0; m<identifications.getLength();m++){
 374                                         if(identifications.item(m).getNodeName().equals("Result")){
 375                                                 results = identifications.item(m).getChildNodes();
 376                                                 for(int k=0; k<results.getLength();k++){
 377                                                         if (results.item(k).getNodeName().equals("TaxonIdentified")){
 378                                                                 tmpName=this.getScientificName(results.item(k), dataHolder);
 379                                                         }
 380                                                 }
 381                                         }
 382                                         if(identifications.item(m).getNodeName().equals("PreferredFlag")){
 383                                                 hasPref=true;
 384                                         }
 385                                 }
 386                                 if ( !hasPref && tmpName != null){
 387                                         if (dataHolder.nomenclatureCode != null && dataHolder.nomenclatureCode !=""){
 388                                                 dataHolder.identificationList.add(tmpName+"_preferred_"+"0"+"_code_" + dataHolder.nomenclatureCode);
 389                                         } else {
 390                                                 dataHolder.identificationList.add(tmpName+"_preferred_"+"0");
 391                                         }
 392                                 }
 393                         }
 394                 }
 395         }
 396
 397
 398
 399         private void getReferences(Node result, Abcd206DataHolder dataHolder){
 400                 NodeList results,reference;
 401                 results = result.getChildNodes();
 402                 for(int k=0; k<results.getLength();k++){
 403                         if (results.item(k).getNodeName().equals("Reference")){
 404                                 reference = results.item(k).getChildNodes();
 405                                 for(int l=0;l<reference.getLength();l++){
 406                                         if (reference.item(l).getNodeName().equals("TitleCitation")){
 407                                                 path = reference.item(l).getNodeName();
 408                                                 dataHolder.referenceList.add(reference.item(l).getTextContent());
 409                                                 getHierarchie(reference.item(l));
 410                                                 dataHolder.knownABCDelements.add(path);
 411                                                 path="";
 412                                         }
 413                                 }
 414                         }
 415                 }
 416         }
 417
 418         private String getScientificName(Node result, Abcd206DataHolder dataHolder){
 419                 NodeList taxonsIdentified, scnames, atomised;
 420                 String tmpName = "";
 421                 dataHolder.atomisedStr = "";
 422                 taxonsIdentified = result.getChildNodes();
 423                 for (int l=0; l<taxonsIdentified.getLength(); l++){
 424                         if (taxonsIdentified.item(l).getNodeName().equals("ScientificName")){
 425                                 scnames = taxonsIdentified.item(l).getChildNodes();
 426                                 for (int n=0;n<scnames.getLength();n++){
 427                                         if (scnames.item(n).getNodeName().equals("FullScientificNameString")){
 428                                                 path=scnames.item(n).getNodeName();
 429                                                 tmpName = scnames.item(n).getTextContent();
 430                                                 getHierarchie(scnames.item(n));
 431                                                 dataHolder.knownABCDelements.add(path);
 432                                                 path="";
 433                                         }
 434                                         if (scnames.item(n).getNodeName().equals("NameAtomised")){
 435                                                 try {
 436                                                         if (scnames.item(n).hasChildNodes()){
 437                                                                 dataHolder.nomenclatureCode = scnames.item(n).getChildNodes().item(1).getNodeName();
 438                                                         }
 439                                                 } catch (Exception e) {
 440                                                         dataHolder.nomenclatureCode ="";
 441                                                 }
 442                                                 atomised = scnames.item(n).getChildNodes().item(1).getChildNodes();
 443                                                 dataHolder.atomisedIdentificationList.add(this.getAtomisedNames(dataHolder.nomenclatureCode, atomised, dataHolder));
 444                                         }
 445                                 }
 446                         }
 447                 }
 448                 return tmpName;
 449         }
 450
 451         private HashMap<String,String> getAtomisedNames(String code, NodeList atomised, Abcd206DataHolder dataHolder){
 452                 if (code.equals("Botanical")){
 453                         return this.getAtomisedBotanical(atomised, dataHolder);
 454                 }
 455                 if (code.equals("Bacterial")){
 456                         return this.getAtomisedBacterial(atomised, dataHolder);
 457                 }
 458                 if (code.equals("NameViral")){
 459                         return this.getAtomisedViral(atomised, dataHolder);
 460                 }
 461                 if (code.equals("NameZoological")){
 462                         return this.getAtomisedZoological(atomised,dataHolder);
 463                 }
 464                 return new HashMap<String,String>();
 465         }
 466
 467         private HashMap<String,String> getAtomisedZoological(NodeList atomised, Abcd206DataHolder dataHolder){
 468                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 469                 for (int i=0;i<atomised.getLength();i++){
 470                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 471                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 472                                 path=atomised.item(i).getNodeName();
 473                                 getHierarchie(atomised.item(i));
 474                                 dataHolder.knownABCDelements.add(path);
 475                                 path="";
 476                         }
 477                         if(atomised.item(i).getNodeName().equals("Subgenus")){
 478                                 atomisedMap.put("Subgenus",atomised.item(i).getTextContent());
 479                                 path=atomised.item(i).getNodeName();
 480                                 getHierarchie(atomised.item(i));
 481                                 dataHolder.knownABCDelements.add(path);
 482                                 path="";
 483                         }
 484                         if(atomised.item(i).getNodeName().equals("SpeciesEpithet")){
 485                                 atomisedMap.put("SpeciesEpithet",atomised.item(i).getTextContent());
 486                                 path=atomised.item(i).getNodeName();
 487                                 getHierarchie(atomised.item(i));
 488                                 dataHolder.knownABCDelements.add(path);
 489                                 path="";
 490                         }
 491                         if(atomised.item(i).getNodeName().equals("SubspeciesEpithet")){
 492                                 atomisedMap.put("SubspeciesEpithet",atomised.item(i).getTextContent());
 493                                 path=atomised.item(i).getNodeName();
 494                                 getHierarchie(atomised.item(i));
 495                                 dataHolder.knownABCDelements.add(path);
 496                                 path="";
 497                         }
 498                         if(atomised.item(i).getNodeName().equals("AuthorTeamOriginalAndYear")){
 499                                 atomisedMap.put("AuthorTeamOriginalAndYear",atomised.item(i).getTextContent());
 500                                 path=atomised.item(i).getNodeName();
 501                                 getHierarchie(atomised.item(i));
 502                                 dataHolder.knownABCDelements.add(path);
 503                                 path="";
 504                         }
 505                         if(atomised.item(i).getNodeName().equals("AuthorTeamParenthesisAndYear")){
 506                                 atomisedMap.put("AuthorTeamParenthesisAndYear",atomised.item(i).getTextContent());
 507                                 path=atomised.item(i).getNodeName();
 508                                 getHierarchie(atomised.item(i));
 509                                 dataHolder.knownABCDelements.add(path);
 510                                 path="";
 511                         }
 512                         if(atomised.item(i).getNodeName().equals("CombinationAuthorTeamAndYear")){
 513                                 atomisedMap.put("CombinationAuthorTeamAndYear",atomised.item(i).getTextContent());
 514                                 path=atomised.item(i).getNodeName();
 515                                 getHierarchie(atomised.item(i));
 516                                 dataHolder.knownABCDelements.add(path);
 517                                 path="";
 518                         }
 519                         if(atomised.item(i).getNodeName().equals("Breed")){
 520                                 atomisedMap.put("Breed",atomised.item(i).getTextContent());
 521                                 path=atomised.item(i).getNodeName();
 522                                 getHierarchie(atomised.item(i));
 523                                 dataHolder.knownABCDelements.add(path);
 524                                 path="";
 525                         }
 526                         if(atomised.item(i).getNodeName().equals("NamedIndividual")){
 527                                 atomisedMap.put("NamedIndividual",atomised.item(i).getTextContent());
 528                                 path=atomised.item(i).getNodeName();
 529                                 getHierarchie(atomised.item(i));
 530                                 dataHolder.knownABCDelements.add(path);
 531                                 path="";
 532                         }
 533                 }
 534                 return atomisedMap;
 535         }
 536
 537         private HashMap<String,String> getAtomisedViral(NodeList atomised, Abcd206DataHolder dataHolder){
 538                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 539                 for (int i=0;i<atomised.getLength();i++){
 540                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 541                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 542                                 path=atomised.item(i).getNodeName();
 543                                 getHierarchie(atomised.item(i));
 544                                 dataHolder.knownABCDelements.add(path);
 545                                 path="";
 546                         }
 547                         if(atomised.item(i).getNodeName().equals("ViralSpeciesDesignation")){
 548                                 atomisedMap.put("ViralSpeciesDesignation", atomised.item(i).getTextContent());
 549                                 path=atomised.item(i).getNodeName();
 550                                 getHierarchie(atomised.item(i));
 551                                 dataHolder.knownABCDelements.add(path);
 552                                 path="";
 553                         }
 554                         if(atomised.item(i).getNodeName().equals("Acronym")){
 555                                 atomisedMap.put("Acronym",atomised.item(i).getTextContent());
 556                                 path=atomised.item(i).getNodeName();
 557                                 getHierarchie(atomised.item(i));
 558                                 dataHolder.knownABCDelements.add(path);
 559                                 path="";
 560                         }
 561                 }
 562                 return atomisedMap;
 563         }
 564
 565         private HashMap<String,String> getAtomisedBotanical(NodeList atomised, Abcd206DataHolder dataHolder){
 566                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 567                 for (int i=0;i<atomised.getLength();i++){
 568                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 569                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 570                                 path=atomised.item(i).getNodeName();
 571                                 getHierarchie(atomised.item(i));
 572                                 dataHolder.knownABCDelements.add(path);
 573                                 path="";
 574                         }
 575                         if(atomised.item(i).getNodeName().equals("FirstEpithet")){
 576                                 atomisedMap.put("FirstEpithet",atomised.item(i).getTextContent());
 577                                 path=atomised.item(i).getNodeName();
 578                                 getHierarchie(atomised.item(i));
 579                                 dataHolder.knownABCDelements.add(path);
 580                                 path="";
 581                         }
 582                         if(atomised.item(i).getNodeName().equals("InfraspecificEpithet")){
 583                                 atomisedMap.put("InfraSpeEpithet", atomised.item(i).getTextContent());
 584                                 path=atomised.item(i).getNodeName();
 585                                 getHierarchie(atomised.item(i));
 586                                 dataHolder.knownABCDelements.add(path);
 587                                 path="";
 588                         }
 589                         if(atomised.item(i).getNodeName().equals("Rank")){
 590                                 atomisedMap.put("Rank",atomised.item(i).getTextContent());
 591                                 path=atomised.item(i).getNodeName();
 592                                 getHierarchie(atomised.item(i));
 593                                 dataHolder.knownABCDelements.add(path);
 594                                 path="";
 595                         }
 596                         if(atomised.item(i).getNodeName().equals("HybridFlag")){
 597                                 atomisedMap.put("HybridFlag",atomised.item(i).getTextContent());
 598                                 path=atomised.item(i).getNodeName();
 599                                 getHierarchie(atomised.item(i));
 600                                 dataHolder.knownABCDelements.add(path);
 601                                 path="";
 602                         }
 603                         if(atomised.item(i).getNodeName().equals("AuthorTeamParenthesis")){
 604                                 atomisedMap.put("AuthorTeamParenthesis",atomised.item(i).getTextContent());
 605                                 path=atomised.item(i).getNodeName();
 606                                 getHierarchie(atomised.item(i));
 607                                 dataHolder.knownABCDelements.add(path);
 608                                 path="";
 609                         }
 610                         if(atomised.item(i).getNodeName().equals("AuthorTeam")){
 611                                 atomisedMap.put("AuthorTeam",atomised.item(i).getTextContent());
 612                                 path=atomised.item(i).getNodeName();
 613                                 getHierarchie(atomised.item(i));
 614                                 dataHolder.knownABCDelements.add(path);
 615                                 path="";
 616                         }
 617                         if(atomised.item(i).getNodeName().equals("CultivarGroupName")){
 618                                 atomisedMap.put("CultivarGroupName",atomised.item(i).getTextContent());
 619                                 path=atomised.item(i).getNodeName();
 620                                 getHierarchie(atomised.item(i));
 621                                 dataHolder.knownABCDelements.add(path);
 622                                 path="";
 623                         }
 624                         if(atomised.item(i).getNodeName().equals("CultivarName")){
 625                                 atomisedMap.put("CultivarName",atomised.item(i).getTextContent());
 626                                 path=atomised.item(i).getNodeName();
 627                                 getHierarchie(atomised.item(i));
 628                                 dataHolder.knownABCDelements.add(path);
 629                                 path="";
 630                         }
 631                         if(atomised.item(i).getNodeName().equals("TradeDesignationNames")){
 632                                 atomisedMap.put("Trade",atomised.item(i).getTextContent());
 633                                 path=atomised.item(i).getNodeName();
 634                                 getHierarchie(atomised.item(i));
 635                                 dataHolder.knownABCDelements.add(path);
 636                                 path="";
 637                         }
 638                 }
 639                 return atomisedMap;
 640         }
 641
 642         private HashMap<String,String> getAtomisedBacterial(NodeList atomised, Abcd206DataHolder dataHolder){
 643                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 644                 for (int i=0;i<atomised.getLength();i++){
 645                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 646                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 647                                 path=atomised.item(i).getNodeName();
 648                                 getHierarchie(atomised.item(i));
 649                                 dataHolder.knownABCDelements.add(path);
 650                                 path="";
 651                         }
 652                         if(atomised.item(i).getNodeName().equals("Subgenus")){
 653                                 atomisedMap.put("SubGenus",atomised.item(i).getTextContent());
 654                                 path=atomised.item(i).getNodeName();
 655                                 getHierarchie(atomised.item(i));
 656                                 dataHolder.knownABCDelements.add(path);
 657                                 path="";
 658                         }
 659                         if(atomised.item(i).getNodeName().equals("SubgenusAuthorAndYear")){
 660                                 atomisedMap.put("SubgenusAuthorAndYear",atomised.item(i).getTextContent());
 661                                 path=atomised.item(i).getNodeName();
 662                                 getHierarchie(atomised.item(i));
 663                                 dataHolder.knownABCDelements.add(path);
 664                                 path="";
 665                         }
 666                         if(atomised.item(i).getNodeName().equals("SpeciesEpithet")){
 667                                 atomisedMap.put("SpeciesEpithet",atomised.item(i).getTextContent());
 668                                 path=atomised.item(i).getNodeName();
 669                                 getHierarchie(atomised.item(i));
 670                                 dataHolder.knownABCDelements.add(path);
 671                                 path="";
 672                         }
 673                         if(atomised.item(i).getNodeName().equals("SubspeciesEpithet")){
 674                                 atomisedMap.put("SubspeciesEpithet",atomised.item(i).getTextContent());
 675                                 path=atomised.item(i).getNodeName();
 676                                 getHierarchie(atomised.item(i));
 677                                 dataHolder.knownABCDelements.add(path);
 678                                 path="";
 679                         }
 680                         if(atomised.item(i).getNodeName().equals("ParentheticalAuthorTeamAndYear")){
 681                                 atomisedMap.put("ParentheticalAuthorTeamAndYear",atomised.item(i).getTextContent());
 682                                 path=atomised.item(i).getNodeName();
 683                                 getHierarchie(atomised.item(i));
 684                                 dataHolder.knownABCDelements.add(path);
 685                                 path="";
 686                         }
 687                         if(atomised.item(i).getNodeName().equals("AuthorTeamAndYear")){
 688                                 atomisedMap.put("AuthorTeamAndYear",atomised.item(i).getTextContent());
 689                                 path=atomised.item(i).getNodeName();
 690                                 getHierarchie(atomised.item(i));
 691                                 dataHolder.knownABCDelements.add(path);
 692                                 path="";
 693                         }
 694                         if(atomised.item(i).getNodeName().equals("NameApprobation")){
 695                                 atomisedMap.put("NameApprobation",atomised.item(i).getTextContent());
 696                                 path=atomised.item(i).getNodeName();
 697                                 getHierarchie(atomised.item(i));
 698                                 dataHolder.knownABCDelements.add(path);
 699                                 path="";
 700                         }
 701                 }
 702                 return atomisedMap;
 703         }
 704
 705         private void getIDs(Element root, Abcd206DataHolder dataHolder){
 706                 NodeList group;
 707                 try {
 708                         group = root.getElementsByTagName("SourceInstitutionID");
 709                         path=group.item(0).getNodeName();
 710                         getHierarchie(group.item(0));
 711                         dataHolder.knownABCDelements.add(path);
 712                         path="";
 713                         dataHolder.institutionCode = group.item(0).getTextContent();
 714                 } catch (NullPointerException e) {
 715                         dataHolder.institutionCode= "";
 716                 }
 717                 try {
 718                         group = root.getElementsByTagName("SourceID");
 719                         path=group.item(0).getNodeName();
 720                         getHierarchie(group.item(0));
 721                         dataHolder.knownABCDelements.add(path);
 722                         path="";
 723                         dataHolder.collectionCode = group.item(0).getTextContent();
 724                 } catch (NullPointerException e) {
 725                         dataHolder.collectionCode = "";
 726                 }
 727                 try {
 728                         group = root.getElementsByTagName("UnitID");
 729                         path=group.item(0).getNodeName();
 730                         getHierarchie(group.item(0));
 731                         dataHolder.knownABCDelements.add(path);
 732                         path="";
 733                         dataHolder.unitID = group.item(0).getTextContent();
 734                 } catch (NullPointerException e) {
 735                         dataHolder.unitID = "";
 736                 }
 737         }
 738
 739         private void getRecordBasis(Element root, Abcd206DataHolder dataHolder){
 740                 NodeList group;
 741                 try {
 742                         group = root.getElementsByTagName("RecordBasis");
 743                         path=group.item(0).getNodeName();
 744                         getHierarchie(group.item(0));
 745                         dataHolder.knownABCDelements.add(path);
 746                         path="";
 747                         dataHolder.recordBasis = group.item(0).getTextContent();
 748                 } catch (NullPointerException e) {
 749                         dataHolder.recordBasis = "";
 750                 }
 751         }
 752
 753         private void getMultimedia(Element root, Abcd206DataHolder dataHolder){
 754                 NodeList group, multimedias, multimedia;
 755                 try {
 756                         group = root.getElementsByTagName("MultiMediaObjects");
 757                         for(int i=0;i<group.getLength();i++){
 758                                 multimedias = group.item(i).getChildNodes();
 759                                 for (int j=0;j<multimedias.getLength();j++){
 760                                         if (multimedias.item(j).getNodeName().equals("MultiMediaObject")){
 761                                                 multimedia = multimedias.item(j).getChildNodes();
 762                                                 for (int k=0;k<multimedia.getLength();k++){
 763                                                         if(multimedia.item(k).getNodeName().equals("FileURI")){
 764                                                                 dataHolder.multimediaObjects.add(multimedia.item(k).getTextContent());
 765                                                                 path = multimedia.item(k).getNodeName();
 766                                                                 getHierarchie(multimedia.item(k));
 767                                                                 dataHolder.knownABCDelements.add(path);
 768                                                                 path="";
 769                                                         }
 770                                                 }
 771                                         }
 772                                 }
 773                         }
 774                 } catch (NullPointerException e) {
 775                         logger.info(e);
 776                 }
 777         }
 778
 779         private void getNumbers(Element root, Abcd206DataHolder dataHolder){
 780                 NodeList group;
 781                 try {
 782                         group = root.getElementsByTagName("AccessionNumber");
 783                         path=group.item(0).getNodeName();
 784                         getHierarchie(group.item(0));
 785                         dataHolder.knownABCDelements.add(path);
 786                         path="";
 787                         dataHolder.accessionNumber = group.item(0).getTextContent();
 788                 } catch (NullPointerException e) {
 789                         dataHolder.accessionNumber = "";
 790                 }
 791                 try {
 792                         group = root.getElementsByTagName("CollectorsFieldNumber");
 793                         path=group.item(0).getNodeName();
 794                         getHierarchie(group.item(0));
 795                         dataHolder.knownABCDelements.add(path);
 796                         path="";
 797                         dataHolder.fieldNumber = group.item(0).getTextContent();
 798                 } catch (NullPointerException e) {
 799                         dataHolder.fieldNumber = "";
 800                 }
 801
 802                 try {
 803                         group = root.getElementsByTagName("CollectorsNumber");
 804                         path=group.item(0).getNodeName();
 805                         getHierarchie(group.item(0));
 806                         dataHolder.knownABCDelements.add(path);
 807                         path="";
 808                         dataHolder.collectorsNumber = group.item(0).getTextContent();
 809                 } catch (NullPointerException e) {
 810                         dataHolder.collectorsNumber = "";
 811                 }
 812
 813                 try {
 814                         group = root.getElementsByTagName("AccessionNumber");
 815                         path=group.item(0).getNodeName();
 816                         getHierarchie(group.item(0));
 817                         dataHolder.knownABCDelements.add(path);
 818                         path="";
 819                         dataHolder.accessionNumber = group.item(0).getTextContent();
 820                 } catch (NullPointerException e) {
 821                         dataHolder.accessionNumber = "";
 822                 }
 823         }
 824
 825         private void getGeolocation(Element root, Abcd206DataHolder dataHolder){
 826                 NodeList group, childs;
 827                 try {
 828                         group = root.getElementsByTagName("LocalityText");
 829                         path=group.item(0).getNodeName();
 830                         getHierarchie(group.item(0));
 831                         dataHolder.knownABCDelements.add(path);
 832                         path="";
 833                         dataHolder.locality = group.item(0).getTextContent();
 834                         if (group.item(0).hasAttributes())
 835                                 if (group.item(0).getAttributes().getNamedItem("lang") != null)
 836                                         dataHolder.languageIso = group.item(0).getAttributes().getNamedItem("lang").getTextContent();
 837                 } catch (NullPointerException e) {
 838                         dataHolder.locality = "";
 839                 }
 840                 try {
 841                         group = root.getElementsByTagName("LongitudeDecimal");
 842                         path=group.item(0).getNodeName();
 843                         getHierarchie(group.item(0));
 844                         dataHolder.knownABCDelements.add(path);
 845                         path="";
 846                         dataHolder.longitude = Double.valueOf(group.item(0).getTextContent());
 847                 } catch (NullPointerException e) {
 848                         dataHolder.longitude=null;
 849                 }
 850                 try {
 851                         group = root.getElementsByTagName("LatitudeDecimal");
 852                         path=group.item(0).getNodeName();
 853                         getHierarchie(group.item(0));
 854                         dataHolder.knownABCDelements.add(path);
 855                         path="";
 856                         dataHolder.latitude = Double.valueOf(group.item(0).getTextContent());
 857                 } catch (NullPointerException e) {
 858                         dataHolder.latitude=null;
 859                 }
 860                 try {
 861                         group = root.getElementsByTagName("Country");
 862                         childs = group.item(0).getChildNodes();
 863                         for (int i=0;i<childs.getLength(); i++){
 864                                 if(childs.item(i).getNodeName() == "Name"){
 865                                         path=childs.item(i).getNodeName();
 866                                         getHierarchie(childs.item(i));
 867                                         dataHolder.knownABCDelements.add(path);
 868                                         path="";
 869                                         dataHolder.country = childs.item(i).getTextContent();
 870                                 }
 871                         }
 872                 } catch (NullPointerException e) {
 873                         dataHolder.country = "";
 874                 }
 875                 try {
 876                         group = root.getElementsByTagName("Country");
 877                         childs = group.item(0).getChildNodes();
 878                         for (int i=0;i<childs.getLength(); i++){
 879                                 if(childs.item(i).getNodeName() == "ISO3166Code"){
 880                                         path=childs.item(i).getNodeName();
 881                                         getHierarchie(childs.item(i));
 882                                         dataHolder.knownABCDelements.add(path);
 883                                         path="";
 884                                         dataHolder.isocountry = childs.item(i).getTextContent();
 885                                 }
 886                         }
 887                 } catch (NullPointerException e) {
 888                         dataHolder.isocountry = "";
 889                 }
 890                 try {
 891                         group = root.getElementsByTagName("Altitude");
 892                         for (int i=0;i<group.getLength();i++){
 893                                 childs = group.item(i).getChildNodes();
 894                                 for (int j=0;j<childs.getLength();j++){
 895                                         if (childs.item(j).getNodeName().equals("MeasurementOrFactText")){
 896                                                 path=childs.item(j).getNodeName();
 897                                                 getHierarchie(childs.item(j));
 898                                                 dataHolder.knownABCDelements.add(path);
 899                                                 path="";
 900                                                 dataHolder.altitude = Integer.valueOf(childs.item(j).getTextContent());
 901                                         }
 902                                 }
 903                         }
 904                 } catch (NullPointerException e) {
 905                         dataHolder.altitude = -9999;
 906                 }
 907
 908                 try {
 909                         group = root.getElementsByTagName("Depth");
 910                         path=group.item(0).getNodeName();
 911                         getHierarchie(group.item(0));
 912                         dataHolder.knownABCDelements.add(path);
 913                         path="";
 914                         dataHolder.depth = Integer.valueOf(group.item(0).getTextContent());
 915                 } catch (NullPointerException e) {
 916                         dataHolder.depth = -9999;
 917                 }
 918
 919                 try{
 920                         group = root.getElementsByTagName("NamedArea");
 921                         dataHolder.namedAreaList = new ArrayList<String>();
 922                         for (int i=0;i<group.getLength();i++){
 923                                 childs = group.item(i).getChildNodes();
 924                                 for (int j=0; j<childs.getLength();j++){
 925                                         if (childs.item(j).getNodeName().equals("AreaName")){
 926                                                 path = childs.item(j).getNodeName();
 927                                                 getHierarchie(childs.item(j));
 928                                                 dataHolder.knownABCDelements.add(path);
 929                                                 path="";
 930                                                 dataHolder.namedAreaList.add(childs.item(j).getTextContent());
 931                                         }
 932                                 }
 933                         }
 934                 }catch(NullPointerException e){
 935                         dataHolder.namedAreaList = new ArrayList<String>();
 936                 }
 937         }
 938
 939         private void getGatheringPeople(Element root, Abcd206DataHolder dataHolder){
 940                 NodeList group, childs, person;
 941                 try {
 942                         group = root.getElementsByTagName("GatheringAgent");
 943                         dataHolder.gatheringAgentList = new ArrayList<String>();
 944                         for (int i=0; i< group.getLength(); i++){
 945                                 childs = group.item(i).getChildNodes();
 946                                 for (int j=0; j<childs.getLength();j++){
 947                                         if (childs.item(j).getNodeName().equals("Person")){
 948                                                 person = childs.item(j).getChildNodes();
 949                                                 for (int k=0; k<person.getLength(); k++){
 950                                                         if (person.item(k).getNodeName().equals("FullName")){
 951                                                                 path=person.item(k).getNodeName();
 952                                                                 getHierarchie(person.item(k));
 953                                                                 dataHolder.knownABCDelements.add(path);
 954                                                                 path="";
 955                                                                 dataHolder.gatheringAgentList.add(person.item(k).getTextContent());
 956                                                         }
 957                                                 }
 958                                         }
 959
 960                                 }
 961                         }
 962                 } catch (NullPointerException e) {
 963                         dataHolder.gatheringAgentList = new ArrayList<String>();
 964                 }
 965         }
 966
 967         private Institution getInstitution(String institutionCode, Abcd206ImportConfigurator config, Abcd206DataHolder dataHolder){
 968                 Institution institution;
 969                 List<Institution> institutions;
 970                 try{
 971                         logger.info(dataHolder.institutionCode);
 972                         institutions = getAgentService().searchInstitutionByCode(dataHolder.institutionCode);
 973                 }catch(Exception e){
 974                         institutions=new ArrayList<Institution>();
 975                 }
 976                 if (institutions.size() ==0 || !config.isReUseExistingMetadata()){
 977                         logger.info("Institution (agent) unknown or not allowed to reuse existing metadata");
 978                         //create institution
 979                         institution = Institution.NewInstance();
 980                         institution.setCode(dataHolder.institutionCode);
 981                 }
 982                 else{
 983                         logger.info("Institution (agent) already in the db");
 984                         institution = institutions.get(0);
 985                 }
 986                 return institution;
 987         }
 988
 989         /*
 990          * Look if the Collection does already exists
 991          * @param collectionCode: a string
 992          * @param institution: the current Institution
 993          * @param app
 994          * @return the Collection (existing or new)
 995          */
 996         private Collection getCollection(String collectionCode, Institution institution, Abcd206ImportConfigurator config, Abcd206DataHolder dataHolder){
 997                 Collection collection = Collection.NewInstance();
 998                 List<Collection> collections;
 999                 try{
1000                         collections = getCollectionService().searchByCode(dataHolder.collectionCode);
1001                 }catch(Exception e){
1002                         collections=new ArrayList<Collection>();
1003                 }
1004                 if (collections.size() ==0 || !config.isReUseExistingMetadata()){
1005                         logger.info("Collection not found or do not reuse existing metadata  " + dataHolder.collectionCode);
1006                         //create new collection
1007                         collection.setCode(dataHolder.collectionCode);
1008                         collection.setCodeStandard("GBIF");
1009                         collection.setInstitute(institution);
1010                 }
1011                 else{
1012                         boolean collectionFound=false;
1013                         for (int i=0; i<collections.size(); i++){
1014                                 collection = collections.get(i);
1015                                 try {
1016                                         if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())){
1017                                                 //found a collection with the same code and the same institution
1018                                                 collectionFound=true;
1019                                         }
1020                                 } catch (NullPointerException e) {}
1021                         }
1022                         if (!collectionFound){
1023                                 collection.setCode(dataHolder.collectionCode);
1024                                 collection.setCodeStandard("GBIF");
1025                                 collection.setInstitute(institution);
1026                         }
1027
1028                 }
1029                 return collection;
1030         }
1031
1032         /*
1033          *
1034          * @param app
1035          * @param derivedThing
1036          * @param sec
1037          */
1038         private void handleIdentifications(Abcd206ImportConfigurator config, DerivedUnitFacade facade, ReferenceBase sec, Abcd206DataHolder dataHolder){
1039                 NonViralName<?> taxonName = null;
1040                 String fullScientificNameString;
1041                 Taxon taxon = null;
1042                 DeterminationEvent determinationEvent = null;
1043                 List<TaxonBase> names = null;
1044
1045                 String scientificName="";
1046                 boolean preferredFlag=false;
1047
1048                 for (int i = 0; i < dataHolder.identificationList.size(); i++) {
1049                         fullScientificNameString = dataHolder.identificationList.get(i);
1050                         fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
1051                         if (fullScientificNameString.indexOf("_preferred_") != -1){
1052                                 scientificName = fullScientificNameString.split("_preferred_")[0];
1053                                 String pTmp = fullScientificNameString.split("_preferred_")[1].split("_code_")[0];
1054                                 if (pTmp.equals("1") || pTmp.toLowerCase().indexOf("true") != -1){
1055                                         preferredFlag=true;
1056                                 } else {
1057                                         preferredFlag=false;
1058                                 }
1059                         }
1060                         else{
1061                                 scientificName = fullScientificNameString;
1062                         }
1063                         logger.info(fullScientificNameString);
1064                         if (fullScientificNameString.indexOf("_code_") != -1){
1065                                 dataHolder.nomenclatureCode = fullScientificNameString.split("_code_")[1];
1066                         }
1067                         if (config.isDoAutomaticParsing() || dataHolder.atomisedIdentificationList == null || dataHolder.atomisedIdentificationList.size()==0){
1068                                 taxonName = this.parseScientificName(scientificName, dataHolder);
1069                         } else {
1070                                 if (dataHolder.atomisedIdentificationList != null || dataHolder.atomisedIdentificationList.size()>0){
1071                                         taxonName = this.setTaxonNameByType(dataHolder.atomisedIdentificationList.get(i), scientificName, dataHolder);
1072                                 }
1073                         }
1074                         if(taxonName == null){
1075                                 taxonName = NonViralName.NewInstance(null);
1076                                 taxonName.setFullTitleCache(scientificName);
1077                         }
1078
1079                         // --- cascade through several options in order to find an appropriate taxon ---
1080
1081                         if (config.isDoMatchTaxa()){
1082                                 taxon = getTaxonService().findBestMatchingTaxon(scientificName);
1083                         }
1084
1085                         if (taxon == null && config.isDoReUseTaxon()){
1086                                 try{
1087                                         names = getTaxonService().searchTaxaByName(scientificName, sec);
1088                                         taxon = (Taxon)names.get(0);
1089                                 } catch(Exception e){
1090                                         taxon=null;
1091                                 }
1092                         } else {
1093                                 logger.info("Matching to existing Taxon : " + taxon.getTitleCache());
1094                         }
1095
1096                         if (!config.isDoReUseTaxon() || taxon == null){
1097                                 getNameService().save(taxonName);
1098                                 taxon = Taxon.NewInstance(taxonName, sec); //TODO sec set null
1099                         }
1100
1101 //                      taxonName = NonViralName.NewInstance(null);
1102 //                      taxonName.setFullTitleCache(scientificName);
1103
1104                         // --- taxon is found now ---
1105
1106                         determinationEvent = DeterminationEvent.NewInstance();
1107                         determinationEvent.setTaxon(taxon);
1108                         determinationEvent.setPreferredFlag(preferredFlag);
1109
1110                         for (String strReference : dataHolder.referenceList){
1111
1112                                 ReferenceBase reference = ReferenceFactory.newGeneric();
1113                                 reference.setTitleCache(strReference, true);
1114                                 determinationEvent.addReference(reference);
1115                         }
1116                         facade.addDetermination(determinationEvent);
1117
1118                         if(config.isDoCreateIndividualsAssociations()){
1119                                 TaxonDescription taxonDescription = null;
1120                                 if(config.isDoMatchToExistingDescription()){
1121                                         logger.error("The import option 'DoMatchToExistingDescription' is not yet implemented.");
1122                                 } else {
1123                                         UUID taxonDescriptionUUID = config.getTaxonToDescriptionMap().get(taxon.getUuid()); // rather put in state
1124                                         taxonDescription = (TaxonDescription) getDescriptionService().load(taxonDescriptionUUID);
1125                                         if(taxonDescription == null){
1126                                                 taxonDescription = TaxonDescription.NewInstance(taxon);
1127                                                 config.getTaxonToDescriptionMap().put(taxon.getUuid(), taxonDescription.getUuid());
1128                                                 if(taxonDescriptionUUID == null){
1129                                                         logger.info("Creating new TaxonDescription for " + taxon.getTitleCache());
1130                                                 } else {
1131                                                         logger.fatal("TaxonDescription with UUID " + taxonDescriptionUUID + " not found --> creating a new one.");
1132                                                 }
1133                                         }
1134                                 }
1135                                 IndividualsAssociation individualsAssociation = IndividualsAssociation.NewInstance();
1136                                 individualsAssociation.setAssociatedSpecimenOrObservation(facade.getDerivedUnit());
1137                                 individualsAssociation.setFeature(Feature.INDIVIDUALS_ASSOCIATION());
1138                                 for(ReferenceBase citation : determinationEvent.getReferences()){
1139                                         individualsAssociation.addSource(DescriptionElementSource.NewInstance(null, null, citation, null));
1140                                 }
1141                                 taxonDescription.addElement(individualsAssociation);
1142                                 getDescriptionService().saveOrUpdate(taxonDescription);
1143                         }
1144                 }
1145
1146         }
1147
1148         private NonViralName<?> parseScientificName(String scientificName, Abcd206DataHolder dataHolder){
1149                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1150                 NonViralName<?>taxonName = null;
1151                 boolean problem=false;
1152
1153                 if (dataHolder.nomenclatureCode.toString().equals("Zoological")){
1154                         taxonName = (ZoologicalName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICZN,null);
1155                         if (taxonName.hasProblem()){
1156                                 problem=true;
1157                         }
1158                 }
1159                 if (dataHolder.nomenclatureCode.toString().equals("Botanical")){
1160                         taxonName  = (BotanicalName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICBN,null);
1161                         if (taxonName.hasProblem()){
1162                                 problem=true;;
1163                         }
1164                 }
1165                 if (dataHolder.nomenclatureCode.toString().equals("Bacterial")){
1166                         taxonName = (BacterialName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNB, null);
1167                         if (taxonName.hasProblem()){
1168                                 problem=true;
1169                         }
1170                 }
1171                 if (dataHolder.nomenclatureCode.toString().equals("Cultivar")){
1172                         taxonName = (CultivarPlantName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNCP, null);
1173                         if (taxonName.hasProblem()){
1174                                 problem=true;
1175                         }
1176                 }
1177 //              if (this.nomenclatureCode.toString().equals("Viral")){
1178 //              ViralName taxonName = (ViralName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICVCN(), null);
1179 //              if (taxonName.hasProblem())
1180 //              logger.info("pb ICVCN");
1181 //              }
1182                 //TODO: parsing of ViralNames?
1183                 if(problem){
1184                         taxonName = NonViralName.NewInstance(null);
1185                         taxonName.setTitleCache(scientificName, true);
1186                 }
1187                 return taxonName;
1188
1189         }
1190
1191         private NonViralName<?> setTaxonNameByType(HashMap<String, String> atomisedMap,String fullName, Abcd206DataHolder dataHolder){
1192                 if (dataHolder.nomenclatureCode.equals("Zoological")){
1193                         NonViralName<ZoologicalName> taxonName  = ZoologicalName.NewInstance(null);
1194                         taxonName.setFullTitleCache(fullName, true);
1195                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1196                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"SubGenus"));
1197                         taxonName.setSpecificEpithet(getFromMap(atomisedMap,"SpeciesEpithet"));
1198                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"SubspeciesEpithet"));
1199                         Team team = null;
1200                         if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1201                                 team = Team.NewInstance();
1202                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"), true);
1203                         }else{
1204                                 if (getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1205                                         team = Team.NewInstance();
1206                                         team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamAndYear"), true);
1207                                 }
1208                         }
1209                         if(team != null){
1210                                 taxonName.setBasionymAuthorTeam(team);
1211                         }else{
1212                                 if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1213                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"));
1214                                 } else if (getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1215                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamAndYear"));
1216                                 }
1217                         }
1218                         if(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear") != null){
1219                                 team = Team.NewInstance();
1220                                 team.setTitleCache(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear"), true);
1221                                 taxonName.setCombinationAuthorTeam(team);
1222                         }
1223                         if (taxonName.hasProblem()){
1224                                 logger.info("pb ICZN");
1225                         }else{
1226                                 return taxonName;
1227                         }
1228                 }
1229                 if (dataHolder.nomenclatureCode.equals("Botanical")){
1230                         NonViralName<BotanicalName> taxonName  = BotanicalName.NewInstance(null);
1231                         taxonName.setFullTitleCache(fullName, true);
1232                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1233                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"FirstEpithet"));
1234                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"InfraSpeEpithet"));
1235                         try{taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap,"Rank")));
1236                         }catch(Exception e){}
1237                         Team team = null;
1238                         if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1239                                 team = Team.NewInstance();
1240                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"), true);
1241                                 if(team != null){
1242                                         taxonName.setBasionymAuthorTeam(team);
1243                                 }
1244                         }
1245                         if (getFromMap(atomisedMap,"AuthorTeam") != null){
1246                                 team = Team.NewInstance();
1247                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeam"), true);
1248                                 if(team != null){
1249                                         taxonName.setCombinationAuthorTeam(team);
1250                                 }
1251                         }
1252                         if (team == null)       {
1253                                 if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1254                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"));
1255                                 }else if (getFromMap(atomisedMap,"AuthorTeam") != null){
1256                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeam"));
1257                                 }
1258                         }
1259                         if(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear") != null){
1260                                 team = Team.NewInstance();
1261                                 team.setTitleCache(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear"), true);
1262                                 taxonName.setCombinationAuthorTeam(team);
1263                         }
1264                         if (taxonName.hasProblem()){
1265                                 logger.info("pb ICBN");
1266                         }else {
1267                                 return taxonName;
1268                         }
1269                 }
1270                 if (dataHolder.nomenclatureCode.equals("Bacterial")){
1271                         NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
1272                         taxonName.setFullTitleCache(fullName, true);
1273                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1274                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"SubGenus"));
1275                         taxonName.setSpecificEpithet(getFromMap(atomisedMap,"Species"));
1276                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"SubspeciesEpithet"));
1277                         if(getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1278                                 Team team = Team.NewInstance();
1279                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamAndYear"), true);
1280                                 taxonName.setCombinationAuthorTeam(team);
1281                         }
1282                         if(getFromMap(atomisedMap,"ParentheticalAuthorTeamAndYear") != null){
1283                                 Team team = Team.NewInstance();
1284                                 team.setTitleCache(getFromMap(atomisedMap,"ParentheticalAuthorTeamAndYear"), true);
1285                                 taxonName.setBasionymAuthorTeam(team);
1286                         }
1287                         if (taxonName.hasProblem()){
1288                                 logger.info("pb ICNB");
1289                         }else{
1290                                 return taxonName;
1291                         }
1292                 }
1293                 if (dataHolder.nomenclatureCode.equals("Cultivar")){
1294                         CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
1295
1296                         if (taxonName.hasProblem()){
1297                                 logger.info("pb ICNCP");
1298                         }else {
1299                                 return taxonName;
1300                         }
1301                 }
1302 //              if (this.nomenclatureCode.equals("Viral")){
1303 //              ViralName taxonName = ViralName.NewInstance(null);
1304 //              taxonName.setFullTitleCache(fullName, true);
1305 //              taxonName.setAcronym(getFromMap(atomisedMap,"Acronym"));
1306 //              if (taxonName.hasProblem())
1307 //              logger.info("pb ICVCN");
1308 //              else return taxonName;
1309 //              }
1310                 //TODO ViralName
1311                 NonViralName<?>taxonName = NonViralName.NewInstance(null);
1312                 taxonName.setFullTitleCache(fullName, true);
1313                 return taxonName;
1314         }
1315
1316         private String getFromMap(HashMap<String, String> atomisedMap, String key){
1317                 String value = null;
1318                 if (atomisedMap.containsKey(key)){
1319                         value = atomisedMap.get(key);
1320                 }
1321                 try{
1322                         if (value != null && key.matches(".*Year.*")){
1323                                 value=value.trim();
1324                                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")){
1325                                         String tmp=value.split("[0-9]{4}$")[0];
1326                                         int year = Integer.parseInt(value.split(tmp)[1]);
1327                                         if (year >= 1752){
1328                                                 value=tmp;
1329                                         }else{
1330                                                 value=null;
1331                                         }
1332                                 }else{
1333                                         value=null;
1334                                 }
1335                         }
1336                 }catch(Exception e){value=null;}
1337
1338                 return value;
1339         }
1340
1341         private void compareABCDtoCDM(URI urlFileName, ArrayList<String> knownElts, Abcd206DataHolder dataHolder){
1342
1343                 try {
1344                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
1345                         DocumentBuilder constructeur = factory.newDocumentBuilder();
1346                         URL url = urlFileName.toURL();
1347                         Object o = url.getContent();
1348                         InputStream is = (InputStream)o;
1349                         Document document = constructeur.parse(is);
1350                         Element root = document.getDocumentElement();
1351                         traverse(root, dataHolder);
1352                 } catch (ParserConfigurationException e) {
1353                         e.printStackTrace();
1354                 } catch (SAXException e) {
1355                         e.printStackTrace();
1356                 } catch (IOException e) {
1357                         e.printStackTrace();
1358                 }
1359                 Set<String> elts = dataHolder.allABCDelements.keySet();
1360                 Iterator< String>it = elts.iterator();
1361                 String elt;
1362                 while (it.hasNext()){
1363                         elt = it.next();
1364                         if (knownElts.indexOf(elt) == -1){
1365                                 logger.info("Unsaved ABCD element: " + elt + " - " + dataHolder.allABCDelements.get(elt));
1366                         }
1367                 }
1368         }
1369
1370
1371
1372         /**
1373          * Traverses the tree for compareABCDtoCDM
1374          * @param node
1375          * @param dataHolder
1376          */
1377         private void traverse(Node node, Abcd206DataHolder dataHolder){
1378                 // Extract node info:
1379                 String test = node.getTextContent();
1380
1381                 // Print and continue traversing.
1382                 if(test != null && test != "#text" && node.getNodeName() != "#text" && test.split("\n").length==1 && test.length()>0){
1383                         path=node.getNodeName();
1384                         getHierarchie(node);
1385                         dataHolder.allABCDelements.put(path,test);
1386                         path="";
1387                 }
1388                 // Now traverse the rest of the tree in depth-first order.
1389                 if (node.hasChildNodes()) {
1390                         // Get the children in a list.
1391                         NodeList nl = node.getChildNodes();
1392                         // How many of them?
1393                         int size = nl.getLength();
1394                         for (int i=0; i<size; i++){
1395                                 // Recursively traverse each of the children.
1396                                 traverse (nl.item(i), dataHolder);
1397                         }
1398                 }
1399         }
1400
1401
1402
1403         @Override
1404         protected boolean isIgnore(Abcd206ImportState state) {
1405                 //return ! config.isDoNameFacts();
1406                 return false;
1407         }
1408
1409
1410
1411 }