cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.net.URL;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.Iterator;
  18 import java.util.List;
  19 import java.util.Set;
  20 import java.util.UUID;
  21
  22 import javax.xml.parsers.DocumentBuilder;
  23 import javax.xml.parsers.DocumentBuilderFactory;
  24 import javax.xml.parsers.ParserConfigurationException;
  25
  26 import org.apache.log4j.Logger;
  27 import org.springframework.stereotype.Component;
  28 import org.springframework.transaction.TransactionStatus;
  29 import org.w3c.dom.Document;
  30 import org.w3c.dom.Element;
  31 import org.w3c.dom.Node;
  32 import org.w3c.dom.NodeList;
  33 import org.xml.sax.SAXException;
  34
  35 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  36 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  37 import eu.etaxonomy.cdm.common.mediaMetaData.ImageMetaData;
  38 import eu.etaxonomy.cdm.io.common.ICdmIO;
  39 import eu.etaxonomy.cdm.io.specimen.SpecimenIoBase;
  40 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  41 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  42 import eu.etaxonomy.cdm.model.agent.Institution;
  43 import eu.etaxonomy.cdm.model.agent.Team;
  44 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  45 import eu.etaxonomy.cdm.model.description.Feature;
  46 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  47 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  48 import eu.etaxonomy.cdm.model.location.NamedArea;
  49 import eu.etaxonomy.cdm.model.media.ImageFile;
  50 import eu.etaxonomy.cdm.model.media.Media;
  51 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  52 import eu.etaxonomy.cdm.model.name.BacterialName;
  53 import eu.etaxonomy.cdm.model.name.BotanicalName;
  54 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  55 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  56 import eu.etaxonomy.cdm.model.name.NonViralName;
  57 import eu.etaxonomy.cdm.model.name.Rank;
  58 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  59 import eu.etaxonomy.cdm.model.occurrence.Collection;
  60 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  61 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  62 import eu.etaxonomy.cdm.model.reference.ReferenceBase;
  63 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  64 import eu.etaxonomy.cdm.model.taxon.Taxon;
  65 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  66 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  67
  68
  69 /**
  70  * @author p.kelbert
  71  * @created 20.10.2008
  72  * @version 1.0
  73  */
  74 @Component
  75 public class Abcd206Import extends SpecimenIoBase<Abcd206ImportConfigurator, Abcd206ImportState> implements ICdmIO<Abcd206ImportState> {
  76         private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  77
  78
  79         public Abcd206Import() {
  80                 super();
  81         }
  82
  83
  84         @Override
  85         protected boolean doCheck(Abcd206ImportState state) {
  86                 logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
  87                 return true;
  88         }
  89
  90
  91         @Override
  92         public boolean doInvoke(Abcd206ImportState state){
  93                 logger.info("INVOKE Specimen Import from ABCD2.06 XML File");
  94                 boolean result = true;
  95                 //AbcdIO test = new AbcdIO();
  96                 String sourceName = state.getConfig().getSource();
  97                 NodeList unitsList = getUnitsNodeList(sourceName);
  98                 if (unitsList != null){
  99                         String message = "nb units to insert: "+unitsList.getLength();
 100                         logger.info(message);
 101                         updateProgress(state, message);
 102
 103                         Abcd206DataHolder dataHolder = new Abcd206DataHolder();
 104
 105                         for (int i=0 ; i<unitsList.getLength() ; i++){
 106                                 this.setUnitPropertiesXML((Element)unitsList.item(i), dataHolder);
 107                                 result &= this.handleSingleUnit(state, dataHolder);
 108
 109                                 //compare the ABCD elements added in to the CDM and the unhandled ABCD elements
 110                                 compareABCDtoCDM(sourceName, dataHolder.knownABCDelements, dataHolder);
 111
 112                                 //reset the ABCD elements added in CDM
 113                                 //knownABCDelements = new ArrayList<String>();
 114                                 dataHolder.allABCDelements = new HashMap<String,String>();
 115                         }
 116                 }
 117
 118                 return result;
 119
 120         }
 121
 122         /*
 123          * Store the unit with its Gathering informations in the CDM
 124          */
 125         private boolean handleSingleUnit(Abcd206ImportState state, Abcd206DataHolder dataHolder){
 126                 boolean result = true;
 127
 128                 Abcd206ImportConfigurator config = state.getConfig();
 129
 130                 TransactionStatus tx = startTransaction();
 131                 try {
 132                         updateProgress(state, "Importing data for unit: " + dataHolder.unitID);
 133
 134 //                      ReferenceBase sec = Database.NewInstance();
 135 //                      sec.setTitleCache("XML DATA");
 136                         ReferenceBase sec = config.getTaxonReference();
 137
 138                         //create facade
 139                         DerivedUnitFacade derivedUnitFacade = getFacade(dataHolder);
 140
 141
 142                         //handle identifications
 143                         handleIdentifications(config, derivedUnitFacade, sec, dataHolder);
 144
 145                         //handle collection data
 146                         setCollectionData(config, dataHolder, derivedUnitFacade);
 147
 148                         /**
 149                          * GATHERING EVENT
 150                          */
 151
 152                         //gathering event
 153                         UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(
 154                                         getTermService(), dataHolder.locality, dataHolder.languageIso, dataHolder.longitude,
 155                                         dataHolder.latitude, dataHolder.gatheringAgentList);
 156
 157                         //country
 158                         UnitsGatheringArea unitsGatheringArea =
 159                                 new UnitsGatheringArea(dataHolder.isocountry, dataHolder.country, getOccurrenceService());
 160                         NamedArea areaCountry = unitsGatheringArea.getArea();
 161
 162                         //other areas
 163                         unitsGatheringArea = new UnitsGatheringArea(dataHolder.namedAreaList);
 164                         ArrayList<NamedArea> nas = unitsGatheringArea.getAreas();
 165                         for (NamedArea namedArea : nas){
 166                                 unitsGatheringEvent.addArea(namedArea);
 167                         }
 168
 169                         //copy gathering event to facade
 170                         GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 171                         derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 172                         derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 173                         derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 174                         derivedUnitFacade.setCountry(areaCountry);
 175                         derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 176
 177                         //TODO exsiccatum
 178
 179
 180                         //add fieldNumber
 181                         derivedUnitFacade.setFieldNumber(dataHolder.fieldNumber);
 182
 183                         //join gatheringEvent to fieldObservation
 184
 185 //                      //add Multimedia URLs
 186                         if(dataHolder.multimediaObjects.size() > 0){
 187                                 MediaRepresentation representation;
 188                                 Media media;
 189                                 ImageMetaData imd ;
 190                                 URL url ;
 191                                 ImageFile imf;
 192                                 for (String multimediaObject : dataHolder.multimediaObjects){
 193                                         if( multimediaObject != null){
 194                                                 imd = ImageMetaData.newInstance();
 195                                                 url = new URL(multimediaObject);
 196                                                 try {
 197                                                         imd.readMetaData(url.toURI(), 0);
 198                                                 } catch (Exception e) {
 199                                                         String message = "An error occurred when trying to read image meta data: " +  e.getMessage();
 200                                                         logger.warn(message);
 201                                                 }
 202                                                 //TODO do we really want to check the url?
 203                                                 if (imd != null){
 204                                                         imf = ImageFile.NewInstance(multimediaObject, null, imd);
 205                                                         representation = MediaRepresentation.NewInstance();
 206                                                         representation.addRepresentationPart(imf);
 207                                                         media = Media.NewInstance();
 208                                                         media.addRepresentation(representation);
 209
 210                                                         derivedUnitFacade.addFieldObjectMedia(media);
 211                                                 }
 212                                         }
 213                                 }
 214                         }
 215
 216                         /**
 217                          * SAVE AND STORE DATA
 218                          */
 219                         getTermService().save(areaCountry);//TODO save area sooner
 220                         for (NamedArea area : nas){
 221                                 getTermService().save(area);//save it sooner (foreach area)
 222                         }
 223                         getTermService().saveLanguageData(unitsGatheringEvent.getLocality());//TODO needs to be saved ?? save it sooner
 224
 225                         getOccurrenceService().save(derivedUnitFacade.getDerivedUnit());
 226                         logger.info("saved ABCD specimen ...");
 227
 228
 229                 } catch (Exception e) {
 230                         logger.warn("Error when reading record!!");
 231                         e.printStackTrace();
 232                         result = false;
 233                 }
 234                 commitTransaction(tx);
 235
 236                 return result;
 237         }
 238
 239
 240         private void setCollectionData(Abcd206ImportConfigurator config,
 241                         Abcd206DataHolder dataHolder, DerivedUnitFacade derivedUnitFacade) {
 242                 //set catalogue number (unitID)
 243                 derivedUnitFacade.setCatalogNumber(dataHolder.unitID);
 244                 derivedUnitFacade.setAccessionNumber(dataHolder.accessionNumber);
 245                 derivedUnitFacade.setCollectorsNumber(dataHolder.collectorsNumber);
 246
 247
 248                 /**
 249                  * INSTITUTION & COLLECTION
 250                  */
 251                 //manage institution
 252                 Institution institution = this.getInstitution(dataHolder.institutionCode, config, dataHolder);
 253                 //manage collection
 254                 Collection collection = this.getCollection(dataHolder.collectionCode, institution, config, dataHolder);
 255                 //link specimen & collection
 256                 derivedUnitFacade.setCollection(collection);
 257         }
 258
 259
 260         private DerivedUnitFacade getFacade(Abcd206DataHolder dataHolder) {
 261                 /**
 262                  * SPECIMEN OR OBSERVATION OR LIVING
 263                  */
 264 //                      DerivedUnitBase derivedThing = null;
 265                 DerivedUnitType type = null;
 266
 267                 //create specimen
 268                 if (dataHolder.recordBasis != null){
 269                         if (dataHolder.recordBasis.toLowerCase().startsWith("s")) {//specimen
 270                                 type = DerivedUnitType.Specimen;
 271                         }else if (dataHolder.recordBasis.toLowerCase().startsWith("o")) {//observation
 272                                 type = DerivedUnitType.Observation;
 273                         }else if (dataHolder.recordBasis.toLowerCase().startsWith("l")) {//living -> fossil, herbarium sheet....???
 274                                 type = DerivedUnitType.LivingBeing;
 275                         }
 276                         if (type == null){
 277                                 logger.info("The basis of record does not seem to be known: " + dataHolder.recordBasis);
 278                                 type = DerivedUnitType.DerivedUnit;
 279                         }
 280                         //TODO fossils?
 281                 }else{
 282                         logger.info("The basis of record is null");
 283                         type = DerivedUnitType.DerivedUnit;
 284                 }
 285                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 286                 return derivedUnitFacade;
 287         }
 288
 289
 290         /*
 291          * Return the list of root nodes for an ABCD 2.06 XML file
 292          * @param fileName: the file's location
 293          * @return the list of root nodes ("Unit")
 294          */
 295         private static NodeList getUnitsNodeList(String urlFileName){
 296                 NodeList unitList = null;
 297                 try {
 298                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 299                         DocumentBuilder builder = factory.newDocumentBuilder();
 300                         URL url = new URL(urlFileName);
 301                         Object o = url.getContent();
 302                         InputStream is = (InputStream)o;
 303                         Document document = builder.parse(is);
 304                         Element root = document.getDocumentElement();
 305                         unitList = root.getElementsByTagName("Unit");
 306
 307                 }catch(Exception e){
 308                         logger.warn(e);
 309                 }
 310                 return unitList;
 311         }
 312
 313
 314         /*
 315          * Store the unit's properties into variables
 316          * Look which unit is the preferred one
 317          * Look what kind of name it is supposed to be, for the parsing (Botanical, Zoological)
 318          * @param racine: the root node for a single unit
 319          */
 320         private void setUnitPropertiesXML(Element root, Abcd206DataHolder dataHolder){
 321                 try{
 322                         NodeList group;
 323
 324 //                      try{afficherInfos(racine, 0);}catch (Exception e) {logger.info(e);}
 325                         group = root.getChildNodes();
 326 //                      logger.info("ABCD ELEMENT not stored: "+group.item(i).getNodeName().toString()+" - value: "+group.item(i).getTextContent());
 327                         for (int i = 0; i < group.getLength(); i++){
 328                                 if (group.item(i).getNodeName().equals("Identifications")){
 329                                         group = group.item(i).getChildNodes();
 330                                         break;
 331                                 }
 332                         }
 333                         dataHolder.identificationList = new ArrayList<String>();
 334                         dataHolder.atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
 335                         dataHolder.referenceList = new ArrayList<String>();
 336                         dataHolder.multimediaObjects = new ArrayList<String>();
 337
 338                         this.getScientificNames(group, dataHolder);
 339
 340 //                      logger.info("this.identificationList "+this.identificationList.toString());
 341                         this.getIDs(root, dataHolder);
 342                         this.getRecordBasis(root, dataHolder);
 343                         this.getMultimedia(root, dataHolder);
 344                         this.getNumbers(root, dataHolder);
 345                         this.getGeolocation(root, dataHolder);
 346                         this.getGatheringPeople(root, dataHolder);
 347
 348                 } catch (Exception e) {
 349                         logger.info("Error occured while parsing XML file"+e);
 350                 }
 351         }
 352
 353         String path= "";
 354         private void getHierarchie(Node node){
 355                 while (node != null && node.getNodeName() != "DataSets"){
 356 //                      logger.info(node.getParentNode().getNodeName());
 357                         path = node.getParentNode().getNodeName()+"/"+path;
 358                         node = node.getParentNode();
 359                 }
 360 //              logger.info("path: "+path);
 361         }
 362
 363         private void getScientificNames(NodeList group, Abcd206DataHolder dataHolder){
 364                 NodeList identifications,results;
 365                 String tmpName = null;
 366                 for (int j=0; j< group.getLength(); j++){
 367                         if(group.item(j).getNodeName().equals("Identification")){
 368                                 dataHolder.nomenclatureCode ="";
 369                                 identifications = group.item(j).getChildNodes();
 370                                 for (int m=0; m<identifications.getLength();m++){
 371                                         if(identifications.item(m).getNodeName().equals("Result")){
 372                                                 results = identifications.item(m).getChildNodes();
 373                                                 for(int k=0; k<results.getLength();k++){
 374                                                         if (results.item(k).getNodeName().equals("TaxonIdentified")){
 375                                                                 tmpName=this.getScientificName(results.item(k), dataHolder);
 376                                                         }
 377                                                 }
 378                                         }else if(identifications.item(m).getNodeName().equals("PreferredFlag")){
 379                                                 if (dataHolder.nomenclatureCode != null && dataHolder.nomenclatureCode !=""){
 380                                                         dataHolder.identificationList.add(tmpName+"_preferred_"+identifications.item(m).getTextContent()+"_code_" + dataHolder.nomenclatureCode);
 381                                                 }else{
 382                                                         dataHolder.identificationList.add(tmpName+"_preferred_"+identifications.item(m).getTextContent());
 383                                                 }
 384                                                 path=identifications.item(m).getNodeName();
 385                                                 getHierarchie(identifications.item(m));
 386                                                 dataHolder.knownABCDelements.add(path);
 387                                                 path="";
 388                                         }else if (identifications.item(m).getNodeName().equals("References")){
 389                                                 this.getReferences(identifications.item(m), dataHolder);
 390                                         }
 391                                 }
 392                         }
 393                 }
 394                 boolean hasPref=false;
 395                 for (int j=0; j< group.getLength(); j++){
 396                         if(group.item(j).getNodeName().equals("Identification")){
 397                                 dataHolder.nomenclatureCode ="";
 398                                 identifications = group.item(j).getChildNodes();
 399                                 for (int m=0; m<identifications.getLength();m++){
 400                                         if(identifications.item(m).getNodeName().equals("Result")){
 401                                                 results = identifications.item(m).getChildNodes();
 402                                                 for(int k=0; k<results.getLength();k++){
 403                                                         if (results.item(k).getNodeName().equals("TaxonIdentified")){
 404                                                                 tmpName=this.getScientificName(results.item(k), dataHolder);
 405                                                         }
 406                                                 }
 407                                         }
 408                                         if(identifications.item(m).getNodeName().equals("PreferredFlag")){
 409                                                 hasPref=true;
 410                                         }
 411                                 }
 412                                 if ( !hasPref && tmpName != null){
 413                                         if (dataHolder.nomenclatureCode != null && dataHolder.nomenclatureCode !=""){
 414                                                 dataHolder.identificationList.add(tmpName+"_preferred_"+"0"+"_code_" + dataHolder.nomenclatureCode);
 415                                         } else {
 416                                                 dataHolder.identificationList.add(tmpName+"_preferred_"+"0");
 417                                         }
 418                                 }
 419                         }
 420                 }
 421         }
 422
 423
 424
 425         private void getReferences(Node result, Abcd206DataHolder dataHolder){
 426                 NodeList results,reference;
 427                 results = result.getChildNodes();
 428                 for(int k=0; k<results.getLength();k++){
 429                         if (results.item(k).getNodeName().equals("Reference")){
 430                                 reference = results.item(k).getChildNodes();
 431                                 for(int l=0;l<reference.getLength();l++){
 432                                         if (reference.item(l).getNodeName().equals("TitleCitation")){
 433                                                 path = reference.item(l).getNodeName();
 434                                                 dataHolder.referenceList.add(reference.item(l).getTextContent());
 435                                                 getHierarchie(reference.item(l));
 436                                                 dataHolder.knownABCDelements.add(path);
 437                                                 path="";
 438                                         }
 439                                 }
 440                         }
 441                 }
 442         }
 443
 444         private String getScientificName(Node result, Abcd206DataHolder dataHolder){
 445                 NodeList taxonsIdentified, scnames, atomised;
 446                 String tmpName = "";
 447                 dataHolder.atomisedStr = "";
 448                 taxonsIdentified = result.getChildNodes();
 449                 for (int l=0; l<taxonsIdentified.getLength(); l++){
 450                         if (taxonsIdentified.item(l).getNodeName().equals("ScientificName")){
 451                                 scnames = taxonsIdentified.item(l).getChildNodes();
 452                                 for (int n=0;n<scnames.getLength();n++){
 453                                         if (scnames.item(n).getNodeName().equals("FullScientificNameString")){
 454                                                 path=scnames.item(n).getNodeName();
 455                                                 tmpName = scnames.item(n).getTextContent();
 456                                                 getHierarchie(scnames.item(n));
 457                                                 dataHolder.knownABCDelements.add(path);
 458                                                 path="";
 459                                         }
 460                                         if (scnames.item(n).getNodeName().equals("NameAtomised")){
 461                                                 try {
 462                                                         if (scnames.item(n).hasChildNodes()){
 463                                                                 dataHolder.nomenclatureCode = scnames.item(n).getChildNodes().item(1).getNodeName();
 464                                                         }
 465                                                 } catch (Exception e) {
 466                                                         dataHolder.nomenclatureCode ="";
 467                                                 }
 468                                                 atomised = scnames.item(n).getChildNodes().item(1).getChildNodes();
 469                                                 dataHolder.atomisedIdentificationList.add(this.getAtomisedNames(dataHolder.nomenclatureCode, atomised, dataHolder));
 470                                         }
 471                                 }
 472                         }
 473                 }
 474                 return tmpName;
 475         }
 476
 477         private HashMap<String,String> getAtomisedNames(String code, NodeList atomised, Abcd206DataHolder dataHolder){
 478                 if (code.equals("Botanical")){
 479                         return this.getAtomisedBotanical(atomised, dataHolder);
 480                 }
 481                 if (code.equals("Bacterial")){
 482                         return this.getAtomisedBacterial(atomised, dataHolder);
 483                 }
 484                 if (code.equals("NameViral")){
 485                         return this.getAtomisedViral(atomised, dataHolder);
 486                 }
 487                 if (code.equals("NameZoological")){
 488                         return this.getAtomisedZoological(atomised,dataHolder);
 489                 }
 490                 return new HashMap<String,String>();
 491         }
 492
 493         private HashMap<String,String> getAtomisedZoological(NodeList atomised, Abcd206DataHolder dataHolder){
 494                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 495                 for (int i=0;i<atomised.getLength();i++){
 496                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 497                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 498                                 path=atomised.item(i).getNodeName();
 499                                 getHierarchie(atomised.item(i));
 500                                 dataHolder.knownABCDelements.add(path);
 501                                 path="";
 502                         }
 503                         if(atomised.item(i).getNodeName().equals("Subgenus")){
 504                                 atomisedMap.put("Subgenus",atomised.item(i).getTextContent());
 505                                 path=atomised.item(i).getNodeName();
 506                                 getHierarchie(atomised.item(i));
 507                                 dataHolder.knownABCDelements.add(path);
 508                                 path="";
 509                         }
 510                         if(atomised.item(i).getNodeName().equals("SpeciesEpithet")){
 511                                 atomisedMap.put("SpeciesEpithet",atomised.item(i).getTextContent());
 512                                 path=atomised.item(i).getNodeName();
 513                                 getHierarchie(atomised.item(i));
 514                                 dataHolder.knownABCDelements.add(path);
 515                                 path="";
 516                         }
 517                         if(atomised.item(i).getNodeName().equals("SubspeciesEpithet")){
 518                                 atomisedMap.put("SubspeciesEpithet",atomised.item(i).getTextContent());
 519                                 path=atomised.item(i).getNodeName();
 520                                 getHierarchie(atomised.item(i));
 521                                 dataHolder.knownABCDelements.add(path);
 522                                 path="";
 523                         }
 524                         if(atomised.item(i).getNodeName().equals("AuthorTeamOriginalAndYear")){
 525                                 atomisedMap.put("AuthorTeamOriginalAndYear",atomised.item(i).getTextContent());
 526                                 path=atomised.item(i).getNodeName();
 527                                 getHierarchie(atomised.item(i));
 528                                 dataHolder.knownABCDelements.add(path);
 529                                 path="";
 530                         }
 531                         if(atomised.item(i).getNodeName().equals("AuthorTeamParenthesisAndYear")){
 532                                 atomisedMap.put("AuthorTeamParenthesisAndYear",atomised.item(i).getTextContent());
 533                                 path=atomised.item(i).getNodeName();
 534                                 getHierarchie(atomised.item(i));
 535                                 dataHolder.knownABCDelements.add(path);
 536                                 path="";
 537                         }
 538                         if(atomised.item(i).getNodeName().equals("CombinationAuthorTeamAndYear")){
 539                                 atomisedMap.put("CombinationAuthorTeamAndYear",atomised.item(i).getTextContent());
 540                                 path=atomised.item(i).getNodeName();
 541                                 getHierarchie(atomised.item(i));
 542                                 dataHolder.knownABCDelements.add(path);
 543                                 path="";
 544                         }
 545                         if(atomised.item(i).getNodeName().equals("Breed")){
 546                                 atomisedMap.put("Breed",atomised.item(i).getTextContent());
 547                                 path=atomised.item(i).getNodeName();
 548                                 getHierarchie(atomised.item(i));
 549                                 dataHolder.knownABCDelements.add(path);
 550                                 path="";
 551                         }
 552                         if(atomised.item(i).getNodeName().equals("NamedIndividual")){
 553                                 atomisedMap.put("NamedIndividual",atomised.item(i).getTextContent());
 554                                 path=atomised.item(i).getNodeName();
 555                                 getHierarchie(atomised.item(i));
 556                                 dataHolder.knownABCDelements.add(path);
 557                                 path="";
 558                         }
 559                 }
 560                 return atomisedMap;
 561         }
 562
 563         private HashMap<String,String> getAtomisedViral(NodeList atomised, Abcd206DataHolder dataHolder){
 564                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 565                 for (int i=0;i<atomised.getLength();i++){
 566                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 567                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 568                                 path=atomised.item(i).getNodeName();
 569                                 getHierarchie(atomised.item(i));
 570                                 dataHolder.knownABCDelements.add(path);
 571                                 path="";
 572                         }
 573                         if(atomised.item(i).getNodeName().equals("ViralSpeciesDesignation")){
 574                                 atomisedMap.put("ViralSpeciesDesignation", atomised.item(i).getTextContent());
 575                                 path=atomised.item(i).getNodeName();
 576                                 getHierarchie(atomised.item(i));
 577                                 dataHolder.knownABCDelements.add(path);
 578                                 path="";
 579                         }
 580                         if(atomised.item(i).getNodeName().equals("Acronym")){
 581                                 atomisedMap.put("Acronym",atomised.item(i).getTextContent());
 582                                 path=atomised.item(i).getNodeName();
 583                                 getHierarchie(atomised.item(i));
 584                                 dataHolder.knownABCDelements.add(path);
 585                                 path="";
 586                         }
 587                 }
 588                 return atomisedMap;
 589         }
 590
 591         private HashMap<String,String> getAtomisedBotanical(NodeList atomised, Abcd206DataHolder dataHolder){
 592                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 593                 for (int i=0;i<atomised.getLength();i++){
 594                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 595                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 596                                 path=atomised.item(i).getNodeName();
 597                                 getHierarchie(atomised.item(i));
 598                                 dataHolder.knownABCDelements.add(path);
 599                                 path="";
 600                         }
 601                         if(atomised.item(i).getNodeName().equals("FirstEpithet")){
 602                                 atomisedMap.put("FirstEpithet",atomised.item(i).getTextContent());
 603                                 path=atomised.item(i).getNodeName();
 604                                 getHierarchie(atomised.item(i));
 605                                 dataHolder.knownABCDelements.add(path);
 606                                 path="";
 607                         }
 608                         if(atomised.item(i).getNodeName().equals("InfraspecificEpithet")){
 609                                 atomisedMap.put("InfraSpeEpithet", atomised.item(i).getTextContent());
 610                                 path=atomised.item(i).getNodeName();
 611                                 getHierarchie(atomised.item(i));
 612                                 dataHolder.knownABCDelements.add(path);
 613                                 path="";
 614                         }
 615                         if(atomised.item(i).getNodeName().equals("Rank")){
 616                                 atomisedMap.put("Rank",atomised.item(i).getTextContent());
 617                                 path=atomised.item(i).getNodeName();
 618                                 getHierarchie(atomised.item(i));
 619                                 dataHolder.knownABCDelements.add(path);
 620                                 path="";
 621                         }
 622                         if(atomised.item(i).getNodeName().equals("HybridFlag")){
 623                                 atomisedMap.put("HybridFlag",atomised.item(i).getTextContent());
 624                                 path=atomised.item(i).getNodeName();
 625                                 getHierarchie(atomised.item(i));
 626                                 dataHolder.knownABCDelements.add(path);
 627                                 path="";
 628                         }
 629                         if(atomised.item(i).getNodeName().equals("AuthorTeamParenthesis")){
 630                                 atomisedMap.put("AuthorTeamParenthesis",atomised.item(i).getTextContent());
 631                                 path=atomised.item(i).getNodeName();
 632                                 getHierarchie(atomised.item(i));
 633                                 dataHolder.knownABCDelements.add(path);
 634                                 path="";
 635                         }
 636                         if(atomised.item(i).getNodeName().equals("AuthorTeam")){
 637                                 atomisedMap.put("AuthorTeam",atomised.item(i).getTextContent());
 638                                 path=atomised.item(i).getNodeName();
 639                                 getHierarchie(atomised.item(i));
 640                                 dataHolder.knownABCDelements.add(path);
 641                                 path="";
 642                         }
 643                         if(atomised.item(i).getNodeName().equals("CultivarGroupName")){
 644                                 atomisedMap.put("CultivarGroupName",atomised.item(i).getTextContent());
 645                                 path=atomised.item(i).getNodeName();
 646                                 getHierarchie(atomised.item(i));
 647                                 dataHolder.knownABCDelements.add(path);
 648                                 path="";
 649                         }
 650                         if(atomised.item(i).getNodeName().equals("CultivarName")){
 651                                 atomisedMap.put("CultivarName",atomised.item(i).getTextContent());
 652                                 path=atomised.item(i).getNodeName();
 653                                 getHierarchie(atomised.item(i));
 654                                 dataHolder.knownABCDelements.add(path);
 655                                 path="";
 656                         }
 657                         if(atomised.item(i).getNodeName().equals("TradeDesignationNames")){
 658                                 atomisedMap.put("Trade",atomised.item(i).getTextContent());
 659                                 path=atomised.item(i).getNodeName();
 660                                 getHierarchie(atomised.item(i));
 661                                 dataHolder.knownABCDelements.add(path);
 662                                 path="";
 663                         }
 664                 }
 665                 return atomisedMap;
 666         }
 667
 668         private HashMap<String,String> getAtomisedBacterial(NodeList atomised, Abcd206DataHolder dataHolder){
 669                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 670                 for (int i=0;i<atomised.getLength();i++){
 671                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 672                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 673                                 path=atomised.item(i).getNodeName();
 674                                 getHierarchie(atomised.item(i));
 675                                 dataHolder.knownABCDelements.add(path);
 676                                 path="";
 677                         }
 678                         if(atomised.item(i).getNodeName().equals("Subgenus")){
 679                                 atomisedMap.put("SubGenus",atomised.item(i).getTextContent());
 680                                 path=atomised.item(i).getNodeName();
 681                                 getHierarchie(atomised.item(i));
 682                                 dataHolder.knownABCDelements.add(path);
 683                                 path="";
 684                         }
 685                         if(atomised.item(i).getNodeName().equals("SubgenusAuthorAndYear")){
 686                                 atomisedMap.put("SubgenusAuthorAndYear",atomised.item(i).getTextContent());
 687                                 path=atomised.item(i).getNodeName();
 688                                 getHierarchie(atomised.item(i));
 689                                 dataHolder.knownABCDelements.add(path);
 690                                 path="";
 691                         }
 692                         if(atomised.item(i).getNodeName().equals("SpeciesEpithet")){
 693                                 atomisedMap.put("SpeciesEpithet",atomised.item(i).getTextContent());
 694                                 path=atomised.item(i).getNodeName();
 695                                 getHierarchie(atomised.item(i));
 696                                 dataHolder.knownABCDelements.add(path);
 697                                 path="";
 698                         }
 699                         if(atomised.item(i).getNodeName().equals("SubspeciesEpithet")){
 700                                 atomisedMap.put("SubspeciesEpithet",atomised.item(i).getTextContent());
 701                                 path=atomised.item(i).getNodeName();
 702                                 getHierarchie(atomised.item(i));
 703                                 dataHolder.knownABCDelements.add(path);
 704                                 path="";
 705                         }
 706                         if(atomised.item(i).getNodeName().equals("ParentheticalAuthorTeamAndYear")){
 707                                 atomisedMap.put("ParentheticalAuthorTeamAndYear",atomised.item(i).getTextContent());
 708                                 path=atomised.item(i).getNodeName();
 709                                 getHierarchie(atomised.item(i));
 710                                 dataHolder.knownABCDelements.add(path);
 711                                 path="";
 712                         }
 713                         if(atomised.item(i).getNodeName().equals("AuthorTeamAndYear")){
 714                                 atomisedMap.put("AuthorTeamAndYear",atomised.item(i).getTextContent());
 715                                 path=atomised.item(i).getNodeName();
 716                                 getHierarchie(atomised.item(i));
 717                                 dataHolder.knownABCDelements.add(path);
 718                                 path="";
 719                         }
 720                         if(atomised.item(i).getNodeName().equals("NameApprobation")){
 721                                 atomisedMap.put("NameApprobation",atomised.item(i).getTextContent());
 722                                 path=atomised.item(i).getNodeName();
 723                                 getHierarchie(atomised.item(i));
 724                                 dataHolder.knownABCDelements.add(path);
 725                                 path="";
 726                         }
 727                 }
 728                 return atomisedMap;
 729         }
 730
 731         private void getIDs(Element root, Abcd206DataHolder dataHolder){
 732                 NodeList group;
 733                 try {
 734                         group = root.getElementsByTagName("SourceInstitutionID");
 735                         path=group.item(0).getNodeName();
 736                         getHierarchie(group.item(0));
 737                         dataHolder.knownABCDelements.add(path);
 738                         path="";
 739                         dataHolder.institutionCode = group.item(0).getTextContent();
 740                 } catch (NullPointerException e) {
 741                         dataHolder.institutionCode= "";
 742                 }
 743                 try {
 744                         group = root.getElementsByTagName("SourceID");
 745                         path=group.item(0).getNodeName();
 746                         getHierarchie(group.item(0));
 747                         dataHolder.knownABCDelements.add(path);
 748                         path="";
 749                         dataHolder.collectionCode = group.item(0).getTextContent();
 750                 } catch (NullPointerException e) {
 751                         dataHolder.collectionCode = "";
 752                 }
 753                 try {
 754                         group = root.getElementsByTagName("UnitID");
 755                         path=group.item(0).getNodeName();
 756                         getHierarchie(group.item(0));
 757                         dataHolder.knownABCDelements.add(path);
 758                         path="";
 759                         dataHolder.unitID = group.item(0).getTextContent();
 760                 } catch (NullPointerException e) {
 761                         dataHolder.unitID = "";
 762                 }
 763         }
 764
 765         private void getRecordBasis(Element root, Abcd206DataHolder dataHolder){
 766                 NodeList group;
 767                 try {
 768                         group = root.getElementsByTagName("RecordBasis");
 769                         path=group.item(0).getNodeName();
 770                         getHierarchie(group.item(0));
 771                         dataHolder.knownABCDelements.add(path);
 772                         path="";
 773                         dataHolder.recordBasis = group.item(0).getTextContent();
 774                 } catch (NullPointerException e) {
 775                         dataHolder.recordBasis = "";
 776                 }
 777         }
 778
 779         private void getMultimedia(Element root, Abcd206DataHolder dataHolder){
 780                 NodeList group, multimedias, multimedia;
 781                 try {
 782                         group = root.getElementsByTagName("MultiMediaObjects");
 783                         for(int i=0;i<group.getLength();i++){
 784                                 multimedias = group.item(i).getChildNodes();
 785                                 for (int j=0;j<multimedias.getLength();j++){
 786                                         if (multimedias.item(j).getNodeName().equals("MultiMediaObject")){
 787                                                 multimedia = multimedias.item(j).getChildNodes();
 788                                                 for (int k=0;k<multimedia.getLength();k++){
 789                                                         if(multimedia.item(k).getNodeName().equals("FileURI")){
 790                                                                 dataHolder.multimediaObjects.add(multimedia.item(k).getTextContent());
 791                                                                 path = multimedia.item(k).getNodeName();
 792                                                                 getHierarchie(multimedia.item(k));
 793                                                                 dataHolder.knownABCDelements.add(path);
 794                                                                 path="";
 795                                                         }
 796                                                 }
 797                                         }
 798                                 }
 799                         }
 800                 } catch (NullPointerException e) {
 801                         logger.info(e);
 802                 }
 803         }
 804
 805         private void getNumbers(Element root, Abcd206DataHolder dataHolder){
 806                 NodeList group;
 807                 try {
 808                         group = root.getElementsByTagName("AccessionNumber");
 809                         path=group.item(0).getNodeName();
 810                         getHierarchie(group.item(0));
 811                         dataHolder.knownABCDelements.add(path);
 812                         path="";
 813                         dataHolder.accessionNumber = group.item(0).getTextContent();
 814                 } catch (NullPointerException e) {
 815                         dataHolder.accessionNumber = "";
 816                 }
 817                 try {
 818                         group = root.getElementsByTagName("CollectorsFieldNumber");
 819                         path=group.item(0).getNodeName();
 820                         getHierarchie(group.item(0));
 821                         dataHolder.knownABCDelements.add(path);
 822                         path="";
 823                         dataHolder.fieldNumber = group.item(0).getTextContent();
 824                 } catch (NullPointerException e) {
 825                         dataHolder.fieldNumber = "";
 826                 }
 827
 828                 try {
 829                         group = root.getElementsByTagName("CollectorsNumber");
 830                         path=group.item(0).getNodeName();
 831                         getHierarchie(group.item(0));
 832                         dataHolder.knownABCDelements.add(path);
 833                         path="";
 834                         dataHolder.collectorsNumber = group.item(0).getTextContent();
 835                 } catch (NullPointerException e) {
 836                         dataHolder.collectorsNumber = "";
 837                 }
 838
 839                 try {
 840                         group = root.getElementsByTagName("AccessionNumber");
 841                         path=group.item(0).getNodeName();
 842                         getHierarchie(group.item(0));
 843                         dataHolder.knownABCDelements.add(path);
 844                         path="";
 845                         dataHolder.accessionNumber = group.item(0).getTextContent();
 846                 } catch (NullPointerException e) {
 847                         dataHolder.accessionNumber = "";
 848                 }
 849         }
 850
 851         private void getGeolocation(Element root, Abcd206DataHolder dataHolder){
 852                 NodeList group, childs;
 853                 try {
 854                         group = root.getElementsByTagName("LocalityText");
 855                         path=group.item(0).getNodeName();
 856                         getHierarchie(group.item(0));
 857                         dataHolder.knownABCDelements.add(path);
 858                         path="";
 859                         dataHolder.locality = group.item(0).getTextContent();
 860                         if (group.item(0).hasAttributes())
 861                                 if (group.item(0).getAttributes().getNamedItem("lang") != null)
 862                                         dataHolder.languageIso = group.item(0).getAttributes().getNamedItem("lang").getTextContent();
 863                 } catch (NullPointerException e) {
 864                         dataHolder.locality = "";
 865                 }
 866                 try {
 867                         group = root.getElementsByTagName("LongitudeDecimal");
 868                         path=group.item(0).getNodeName();
 869                         getHierarchie(group.item(0));
 870                         dataHolder.knownABCDelements.add(path);
 871                         path="";
 872                         dataHolder.longitude = Double.valueOf(group.item(0).getTextContent());
 873                 } catch (NullPointerException e) {
 874                         dataHolder.longitude=0.0;
 875                 }
 876                 try {
 877                         group = root.getElementsByTagName("LatitudeDecimal");
 878                         path=group.item(0).getNodeName();
 879                         getHierarchie(group.item(0));
 880                         dataHolder.knownABCDelements.add(path);
 881                         path="";
 882                         dataHolder.latitude = Double.valueOf(group.item(0).getTextContent());
 883                 } catch (NullPointerException e) {
 884                         dataHolder.latitude=0.0;
 885                 }
 886                 try {
 887                         group = root.getElementsByTagName("Country");
 888                         childs = group.item(0).getChildNodes();
 889                         for (int i=0;i<childs.getLength(); i++){
 890                                 if(childs.item(i).getNodeName() == "Name"){
 891                                         path=childs.item(i).getNodeName();
 892                                         getHierarchie(childs.item(i));
 893                                         dataHolder.knownABCDelements.add(path);
 894                                         path="";
 895                                         dataHolder.country = childs.item(i).getTextContent();
 896                                 }
 897                         }
 898                 } catch (NullPointerException e) {
 899                         dataHolder.country = "";
 900                 }
 901                 try {
 902                         group = root.getElementsByTagName("Country");
 903                         childs = group.item(0).getChildNodes();
 904                         for (int i=0;i<childs.getLength(); i++){
 905                                 if(childs.item(i).getNodeName() == "ISO3166Code"){
 906                                         path=childs.item(i).getNodeName();
 907                                         getHierarchie(childs.item(i));
 908                                         dataHolder.knownABCDelements.add(path);
 909                                         path="";
 910                                         dataHolder.isocountry = childs.item(i).getTextContent();
 911                                 }
 912                         }
 913                 } catch (NullPointerException e) {
 914                         dataHolder.isocountry = "";
 915                 }
 916                 try {
 917                         group = root.getElementsByTagName("Altitude");
 918                         for (int i=0;i<group.getLength();i++){
 919                                 childs = group.item(i).getChildNodes();
 920                                 for (int j=0;j<childs.getLength();j++){
 921                                         if (childs.item(j).getNodeName().equals("MeasurementOrFactText")){
 922                                                 path=childs.item(j).getNodeName();
 923                                                 getHierarchie(childs.item(j));
 924                                                 dataHolder.knownABCDelements.add(path);
 925                                                 path="";
 926                                                 dataHolder.altitude = Integer.valueOf(childs.item(j).getTextContent());
 927                                         }
 928                                 }
 929                         }
 930                 } catch (NullPointerException e) {
 931                         dataHolder.altitude = -9999;
 932                 }
 933
 934                 try {
 935                         group = root.getElementsByTagName("Depth");
 936                         path=group.item(0).getNodeName();
 937                         getHierarchie(group.item(0));
 938                         dataHolder.knownABCDelements.add(path);
 939                         path="";
 940                         dataHolder.depth = Integer.valueOf(group.item(0).getTextContent());
 941                 } catch (NullPointerException e) {
 942                         dataHolder.depth = -9999;
 943                 }
 944
 945                 try{
 946                         group = root.getElementsByTagName("NamedArea");
 947                         dataHolder.namedAreaList = new ArrayList<String>();
 948                         for (int i=0;i<group.getLength();i++){
 949                                 childs = group.item(i).getChildNodes();
 950                                 for (int j=0; j<childs.getLength();j++){
 951                                         if (childs.item(j).getNodeName().equals("AreaName")){
 952                                                 path = childs.item(j).getNodeName();
 953                                                 getHierarchie(childs.item(j));
 954                                                 dataHolder.knownABCDelements.add(path);
 955                                                 path="";
 956                                                 dataHolder.namedAreaList.add(childs.item(j).getTextContent());
 957                                         }
 958                                 }
 959                         }
 960                 }catch(NullPointerException e){
 961                         dataHolder.namedAreaList = new ArrayList<String>();
 962                 }
 963         }
 964
 965         private void getGatheringPeople(Element root, Abcd206DataHolder dataHolder){
 966                 NodeList group, childs, person;
 967                 try {
 968                         group = root.getElementsByTagName("GatheringAgent");
 969                         dataHolder.gatheringAgentList = new ArrayList<String>();
 970                         for (int i=0; i< group.getLength(); i++){
 971                                 childs = group.item(i).getChildNodes();
 972                                 for (int j=0; j<childs.getLength();j++){
 973                                         if (childs.item(j).getNodeName().equals("Person")){
 974                                                 person = childs.item(j).getChildNodes();
 975                                                 for (int k=0; k<person.getLength(); k++){
 976                                                         if (person.item(k).getNodeName().equals("FullName")){
 977                                                                 path=person.item(k).getNodeName();
 978                                                                 getHierarchie(person.item(k));
 979                                                                 dataHolder.knownABCDelements.add(path);
 980                                                                 path="";
 981                                                                 dataHolder.gatheringAgentList.add(person.item(k).getTextContent());
 982                                                         }
 983                                                 }
 984                                         }
 985
 986                                 }
 987                         }
 988                 } catch (NullPointerException e) {
 989                         dataHolder.gatheringAgentList = new ArrayList<String>();
 990                 }
 991         }
 992
 993         private Institution getInstitution(String institutionCode, Abcd206ImportConfigurator config, Abcd206DataHolder dataHolder){
 994                 Institution institution;
 995                 List<Institution> institutions;
 996                 try{
 997                         logger.info(dataHolder.institutionCode);
 998                         institutions = getAgentService().searchInstitutionByCode(dataHolder.institutionCode);
 999                 }catch(Exception e){
1000                         institutions=new ArrayList<Institution>();
1001                 }
1002                 if (institutions.size() ==0 || !config.isReUseExistingMetadata()){
1003                         logger.info("Institution (agent) unknown or not allowed to reuse existing metadata");
1004                         //create institution
1005                         institution = Institution.NewInstance();
1006                         institution.setCode(dataHolder.institutionCode);
1007                 }
1008                 else{
1009                         logger.info("Institution (agent) already in the db");
1010                         institution = institutions.get(0);
1011                 }
1012                 return institution;
1013         }
1014
1015         /*
1016          * Look if the Collection does already exists
1017          * @param collectionCode: a string
1018          * @param institution: the current Institution
1019          * @param app
1020          * @return the Collection (existing or new)
1021          */
1022         private Collection getCollection(String collectionCode, Institution institution, Abcd206ImportConfigurator config, Abcd206DataHolder dataHolder){
1023                 Collection collection = Collection.NewInstance();
1024                 List<Collection> collections;
1025                 try{
1026                         collections = getCollectionService().searchByCode(dataHolder.collectionCode);
1027                 }catch(Exception e){
1028                         collections=new ArrayList<Collection>();
1029                 }
1030                 if (collections.size() ==0 || !config.isReUseExistingMetadata()){
1031                         logger.info("Collection not found or do not reuse existing metadata  " + dataHolder.collectionCode);
1032                         //create new collection
1033                         collection.setCode(dataHolder.collectionCode);
1034                         collection.setCodeStandard("GBIF");
1035                         collection.setInstitute(institution);
1036                 }
1037                 else{
1038                         boolean collectionFound=false;
1039                         for (int i=0; i<collections.size(); i++){
1040                                 collection = collections.get(i);
1041                                 try {
1042                                         if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())){
1043                                                 //found a collection with the same code and the same institution
1044                                                 collectionFound=true;
1045                                         }
1046                                 } catch (NullPointerException e) {}
1047                         }
1048                         if (!collectionFound){
1049                                 collection.setCode(dataHolder.collectionCode);
1050                                 collection.setCodeStandard("GBIF");
1051                                 collection.setInstitute(institution);
1052                         }
1053
1054                 }
1055                 return collection;
1056         }
1057
1058         /*
1059          *
1060          * @param app
1061          * @param derivedThing
1062          * @param sec
1063          */
1064         private void handleIdentifications(Abcd206ImportConfigurator config, DerivedUnitFacade facade, ReferenceBase sec, Abcd206DataHolder dataHolder){
1065                 NonViralName<?> taxonName = null;
1066                 String fullScientificNameString;
1067                 Taxon taxon = null;
1068                 DeterminationEvent determinationEvent = null;
1069                 List<TaxonBase> names = null;
1070
1071                 String scientificName="";
1072                 boolean preferredFlag=false;
1073
1074                 for (int i = 0; i < dataHolder.identificationList.size(); i++) {
1075                         fullScientificNameString = dataHolder.identificationList.get(i);
1076                         fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
1077                         if (fullScientificNameString.indexOf("_preferred_") != -1){
1078                                 scientificName = fullScientificNameString.split("_preferred_")[0];
1079                                 String pTmp = fullScientificNameString.split("_preferred_")[1].split("_code_")[0];
1080                                 if (pTmp.equals("1") || pTmp.toLowerCase().indexOf("true") != -1){
1081                                         preferredFlag=true;
1082                                 } else {
1083                                         preferredFlag=false;
1084                                 }
1085                         }
1086                         else{
1087                                 scientificName = fullScientificNameString;
1088                         }
1089                         logger.info(fullScientificNameString);
1090                         if (fullScientificNameString.indexOf("_code_") != -1){
1091                                 dataHolder.nomenclatureCode = fullScientificNameString.split("_code_")[1];
1092                         }
1093                         if (config.isDoAutomaticParsing() || dataHolder.atomisedIdentificationList == null || dataHolder.atomisedIdentificationList.size()==0){
1094                                 taxonName = this.parseScientificName(scientificName, dataHolder);
1095                         } else {
1096                                 if (dataHolder.atomisedIdentificationList != null || dataHolder.atomisedIdentificationList.size()>0){
1097                                         taxonName = this.setTaxonNameByType(dataHolder.atomisedIdentificationList.get(i), scientificName, dataHolder);
1098                                 }
1099                         }
1100                         if(taxonName == null){
1101                                 taxonName = NonViralName.NewInstance(null);
1102                                 taxonName.setFullTitleCache(scientificName);
1103                         }
1104
1105                         // --- cascade through several options in order to find an appropriate taxon ---
1106
1107                         if (config.isDoMatchTaxa()){
1108                                 taxon = getTaxonService().findBestMatchingTaxon(scientificName);
1109                         }
1110                         if (taxon == null && config.isDoReUseTaxon()){
1111                                 try{
1112                                         names = getTaxonService().searchTaxaByName(scientificName, sec);
1113                                         taxon = (Taxon)names.get(0);
1114                                 } catch(Exception e){
1115                                         taxon=null;
1116                                 }
1117                         } else {
1118                                 logger.info("Matching to existing Taxon : " + taxon.getTitleCache());
1119                         }
1120
1121                         if (!config.isDoReUseTaxon() || taxon == null){
1122                                 getNameService().save(taxonName);
1123                                 taxon = Taxon.NewInstance(taxonName, sec); //TODO sec set null
1124                         }
1125
1126 //                      taxonName = NonViralName.NewInstance(null);
1127 //                      taxonName.setFullTitleCache(scientificName);
1128
1129                         // --- taxon is found now ---
1130
1131                         determinationEvent = DeterminationEvent.NewInstance();
1132                         determinationEvent.setTaxon(taxon);
1133                         determinationEvent.setPreferredFlag(preferredFlag);
1134
1135                         for (String strReference : dataHolder.referenceList){
1136
1137                                 ReferenceBase reference = ReferenceFactory.newGeneric();
1138                                 reference.setTitleCache(strReference, true);
1139                                 determinationEvent.addReference(reference);
1140                         }
1141                         facade.addDetermination(determinationEvent);
1142
1143                         if(config.isDoCreateIndividualsAssociations()){
1144                                 TaxonDescription taxonDescription = null;
1145                                 if(config.isDoMatchToExistingDescription()){
1146                                         logger.error("The import option 'DoMatchToExistingDescription' is not yet implemented.");
1147                                 } else {
1148                                         UUID taxonDescriptionUUID = config.getTaxonToDescriptionMap().get(taxon.getUuid()); // rather put in state
1149                                         taxonDescription = (TaxonDescription) getDescriptionService().load(taxonDescriptionUUID);
1150                                         if(taxonDescription == null){
1151                                                 taxonDescription = TaxonDescription.NewInstance(taxon);
1152                                                 config.getTaxonToDescriptionMap().put(taxon.getUuid(), taxonDescription.getUuid());
1153                                                 if(taxonDescriptionUUID == null){
1154                                                         logger.info("Creating new TaxonDescription for " + taxon.getTitleCache());
1155                                                 } else {
1156                                                         logger.fatal("TaxonDescription with UUID " + taxonDescriptionUUID + " not found --> creating a new one.");
1157                                                 }
1158                                         }
1159                                 }
1160                                 IndividualsAssociation individualsAssociation = IndividualsAssociation.NewInstance();
1161                                 individualsAssociation.setAssociatedSpecimenOrObservation(facade.getDerivedUnit());
1162                                 individualsAssociation.setFeature(Feature.INDIVIDUALS_ASSOCIATION());
1163                                 for(ReferenceBase citation : determinationEvent.getReferences()){
1164                                         individualsAssociation.addSource(DescriptionElementSource.NewInstance(null, null, citation, null));
1165                                 }
1166                                 taxonDescription.addElement(individualsAssociation);
1167                                 getDescriptionService().saveOrUpdate(taxonDescription);
1168                         }
1169                 }
1170
1171         }
1172
1173         private NonViralName<?> parseScientificName(String scientificName, Abcd206DataHolder dataHolder){
1174                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1175                 NonViralName<?>taxonName = null;
1176                 boolean problem=false;
1177
1178                 if (dataHolder.nomenclatureCode.toString().equals("Zoological")){
1179                         taxonName = (ZoologicalName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICZN,null);
1180                         if (taxonName.hasProblem()){
1181                                 problem=true;
1182                         }
1183                 }
1184                 if (dataHolder.nomenclatureCode.toString().equals("Botanical")){
1185                         taxonName  = (BotanicalName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICBN,null);
1186                         if (taxonName.hasProblem()){
1187                                 problem=true;;
1188                         }
1189                 }
1190                 if (dataHolder.nomenclatureCode.toString().equals("Bacterial")){
1191                         taxonName = (BacterialName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNB, null);
1192                         if (taxonName.hasProblem()){
1193                                 problem=true;
1194                         }
1195                 }
1196                 if (dataHolder.nomenclatureCode.toString().equals("Cultivar")){
1197                         taxonName = (CultivarPlantName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNCP, null);
1198                         if (taxonName.hasProblem()){
1199                                 problem=true;
1200                         }
1201                 }
1202 //              if (this.nomenclatureCode.toString().equals("Viral")){
1203 //              ViralName taxonName = (ViralName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICVCN(), null);
1204 //              if (taxonName.hasProblem())
1205 //              logger.info("pb ICVCN");
1206 //              }
1207                 //TODO: parsing of ViralNames?
1208                 if(problem){
1209                         taxonName = NonViralName.NewInstance(null);
1210                         taxonName.setTitleCache(scientificName, true);
1211                 }
1212                 return taxonName;
1213
1214         }
1215
1216         private NonViralName<?> setTaxonNameByType(HashMap<String, String> atomisedMap,String fullName, Abcd206DataHolder dataHolder){
1217                 if (dataHolder.nomenclatureCode.equals("Zoological")){
1218                         NonViralName<ZoologicalName> taxonName  = ZoologicalName.NewInstance(null);
1219                         taxonName.setFullTitleCache(fullName, true);
1220                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1221                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"SubGenus"));
1222                         taxonName.setSpecificEpithet(getFromMap(atomisedMap,"SpeciesEpithet"));
1223                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"SubspeciesEpithet"));
1224                         Team team = null;
1225                         if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1226                                 team = Team.NewInstance();
1227                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"), true);
1228                         }else{
1229                                 if (getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1230                                         team = Team.NewInstance();
1231                                         team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamAndYear"), true);
1232                                 }
1233                         }
1234                         if(team != null){
1235                                 taxonName.setBasionymAuthorTeam(team);
1236                         }else{
1237                                 if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1238                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"));
1239                                 } else if (getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1240                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamAndYear"));
1241                                 }
1242                         }
1243                         if(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear") != null){
1244                                 team = Team.NewInstance();
1245                                 team.setTitleCache(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear"), true);
1246                                 taxonName.setCombinationAuthorTeam(team);
1247                         }
1248                         if (taxonName.hasProblem()){
1249                                 logger.info("pb ICZN");
1250                         }else{
1251                                 return taxonName;
1252                         }
1253                 }
1254                 if (dataHolder.nomenclatureCode.equals("Botanical")){
1255                         NonViralName<BotanicalName> taxonName  = BotanicalName.NewInstance(null);
1256                         taxonName.setFullTitleCache(fullName, true);
1257                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1258                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"FirstEpithet"));
1259                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"InfraSpeEpithet"));
1260                         try{taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap,"Rank")));
1261                         }catch(Exception e){}
1262                         Team team = null;
1263                         if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1264                                 team = Team.NewInstance();
1265                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"), true);
1266                                 if(team != null){
1267                                         taxonName.setBasionymAuthorTeam(team);
1268                                 }
1269                         }
1270                         if (getFromMap(atomisedMap,"AuthorTeam") != null){
1271                                 team = Team.NewInstance();
1272                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeam"), true);
1273                                 if(team != null){
1274                                         taxonName.setCombinationAuthorTeam(team);
1275                                 }
1276                         }
1277                         if (team == null)       {
1278                                 if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1279                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"));
1280                                 }else if (getFromMap(atomisedMap,"AuthorTeam") != null){
1281                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeam"));
1282                                 }
1283                         }
1284                         if(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear") != null){
1285                                 team = Team.NewInstance();
1286                                 team.setTitleCache(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear"), true);
1287                                 taxonName.setCombinationAuthorTeam(team);
1288                         }
1289                         if (taxonName.hasProblem()){
1290                                 logger.info("pb ICBN");
1291                         }else {
1292                                 return taxonName;
1293                         }
1294                 }
1295                 if (dataHolder.nomenclatureCode.equals("Bacterial")){
1296                         NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
1297                         taxonName.setFullTitleCache(fullName, true);
1298                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1299                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"SubGenus"));
1300                         taxonName.setSpecificEpithet(getFromMap(atomisedMap,"Species"));
1301                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"SubspeciesEpithet"));
1302                         if(getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1303                                 Team team = Team.NewInstance();
1304                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamAndYear"), true);
1305                                 taxonName.setCombinationAuthorTeam(team);
1306                         }
1307                         if(getFromMap(atomisedMap,"ParentheticalAuthorTeamAndYear") != null){
1308                                 Team team = Team.NewInstance();
1309                                 team.setTitleCache(getFromMap(atomisedMap,"ParentheticalAuthorTeamAndYear"), true);
1310                                 taxonName.setBasionymAuthorTeam(team);
1311                         }
1312                         if (taxonName.hasProblem()){
1313                                 logger.info("pb ICNB");
1314                         }else{
1315                                 return taxonName;
1316                         }
1317                 }
1318                 if (dataHolder.nomenclatureCode.equals("Cultivar")){
1319                         CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
1320
1321                         if (taxonName.hasProblem()){
1322                                 logger.info("pb ICNCP");
1323                         }else {
1324                                 return taxonName;
1325                         }
1326                 }
1327 //              if (this.nomenclatureCode.equals("Viral")){
1328 //              ViralName taxonName = ViralName.NewInstance(null);
1329 //              taxonName.setFullTitleCache(fullName, true);
1330 //              taxonName.setAcronym(getFromMap(atomisedMap,"Acronym"));
1331 //              if (taxonName.hasProblem())
1332 //              logger.info("pb ICVCN");
1333 //              else return taxonName;
1334 //              }
1335                 //TODO ViralName
1336                 NonViralName<?>taxonName = NonViralName.NewInstance(null);
1337                 taxonName.setFullTitleCache(fullName, true);
1338                 return taxonName;
1339         }
1340
1341         private String getFromMap(HashMap<String, String> atomisedMap, String key){
1342                 String value = null;
1343                 if (atomisedMap.containsKey(key)){
1344                         value = atomisedMap.get(key);
1345                 }
1346                 try{
1347                         if (value != null && key.matches(".*Year.*")){
1348                                 value=value.trim();
1349                                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")){
1350                                         String tmp=value.split("[0-9]{4}$")[0];
1351                                         int year = Integer.parseInt(value.split(tmp)[1]);
1352                                         if (year >= 1752){
1353                                                 value=tmp;
1354                                         }else{
1355                                                 value=null;
1356                                         }
1357                                 }else{
1358                                         value=null;
1359                                 }
1360                         }
1361                 }catch(Exception e){value=null;}
1362
1363                 return value;
1364         }
1365
1366         private void compareABCDtoCDM(String urlFileName, ArrayList<String> knownElts, Abcd206DataHolder dataHolder){
1367
1368                 try {
1369                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
1370                         DocumentBuilder constructeur = factory.newDocumentBuilder();
1371                         URL url = new URL(urlFileName);
1372                         Object o = url.getContent();
1373                         InputStream is = (InputStream)o;
1374                         Document document = constructeur.parse(is);
1375                         Element root = document.getDocumentElement();
1376                         traverse(root, dataHolder);
1377                 } catch (ParserConfigurationException e) {
1378                         e.printStackTrace();
1379                 } catch (SAXException e) {
1380                         e.printStackTrace();
1381                 } catch (IOException e) {
1382                         e.printStackTrace();
1383                 }
1384                 Set<String> elts = dataHolder.allABCDelements.keySet();
1385                 Iterator< String>it = elts.iterator();
1386                 String elt;
1387                 while (it.hasNext()){
1388                         elt = it.next();
1389                         if (knownElts.indexOf(elt) == -1){
1390                                 logger.info("Unsaved ABCD element: " + elt + " - " + dataHolder.allABCDelements.get(elt));
1391                         }
1392                 }
1393         }
1394
1395
1396
1397         /**
1398          * Traverses the tree for compareABCDtoCDM
1399          * @param node
1400          * @param dataHolder
1401          */
1402         private void traverse(Node node, Abcd206DataHolder dataHolder){
1403                 // Extract node info:
1404                 String test = node.getTextContent();
1405
1406                 // Print and continue traversing.
1407                 if(test != null && test != "#text" && node.getNodeName() != "#text" && test.split("\n").length==1 && test.length()>0){
1408                         path=node.getNodeName();
1409                         getHierarchie(node);
1410                         dataHolder.allABCDelements.put(path,test);
1411                         path="";
1412                 }
1413                 // Now traverse the rest of the tree in depth-first order.
1414                 if (node.hasChildNodes()) {
1415                         // Get the children in a list.
1416                         NodeList nl = node.getChildNodes();
1417                         // How many of them?
1418                         int size = nl.getLength();
1419                         for (int i=0; i<size; i++){
1420                                 // Recursively traverse each of the children.
1421                                 traverse (nl.item(i), dataHolder);
1422                         }
1423                 }
1424         }
1425
1426
1427
1428         @Override
1429         protected boolean isIgnore(Abcd206ImportState state) {
1430                 //return ! config.isDoNameFacts();
1431                 return false;
1432         }
1433
1434
1435
1436 }