cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.net.URL;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.Iterator;
  18 import java.util.List;
  19 import java.util.Set;
  20
  21 import javax.xml.parsers.DocumentBuilder;
  22 import javax.xml.parsers.DocumentBuilderFactory;
  23 import javax.xml.parsers.ParserConfigurationException;
  24
  25 import org.apache.log4j.Logger;
  26 import org.springframework.stereotype.Component;
  27 import org.springframework.transaction.TransactionStatus;
  28 import org.w3c.dom.Document;
  29 import org.w3c.dom.Element;
  30 import org.w3c.dom.Node;
  31 import org.w3c.dom.NodeList;
  32 import org.xml.sax.SAXException;
  33
  34 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  35 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  36 import eu.etaxonomy.cdm.common.mediaMetaData.ImageMetaData;
  37 import eu.etaxonomy.cdm.io.common.ICdmIO;
  38 import eu.etaxonomy.cdm.io.specimen.SpecimenIoBase;
  39 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  40 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  41 import eu.etaxonomy.cdm.model.agent.Institution;
  42 import eu.etaxonomy.cdm.model.agent.Team;
  43 import eu.etaxonomy.cdm.model.location.NamedArea;
  44 import eu.etaxonomy.cdm.model.media.ImageFile;
  45 import eu.etaxonomy.cdm.model.media.Media;
  46 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
  47 import eu.etaxonomy.cdm.model.name.BacterialName;
  48 import eu.etaxonomy.cdm.model.name.BotanicalName;
  49 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  50 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  51 import eu.etaxonomy.cdm.model.name.NonViralName;
  52 import eu.etaxonomy.cdm.model.name.Rank;
  53 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  54 import eu.etaxonomy.cdm.model.occurrence.Collection;
  55 import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
  56 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  57 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
  58 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  59 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
  60 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  61 import eu.etaxonomy.cdm.model.occurrence.LivingBeing;
  62 import eu.etaxonomy.cdm.model.occurrence.Observation;
  63 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  64 import eu.etaxonomy.cdm.model.reference.ReferenceBase;
  65 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  66 import eu.etaxonomy.cdm.model.taxon.Taxon;
  67 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  68 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  69
  70
  71 /**
  72  * @author p.kelbert
  73  * @created 20.10.2008
  74  * @version 1.0
  75  */
  76 @Component
  77 public class Abcd206Import extends SpecimenIoBase<Abcd206ImportConfigurator, Abcd206ImportState> implements ICdmIO<Abcd206ImportState> {
  78         private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  79
  80
  81         public Abcd206Import() {
  82                 super();
  83         }
  84
  85
  86         @Override
  87         protected boolean doCheck(Abcd206ImportState state) {
  88                 logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
  89                 return true;
  90         }
  91
  92
  93         @Override
  94         public boolean doInvoke(Abcd206ImportState state){
  95                 logger.info("INVOKE Specimen Import from ABCD2.06 XML File");
  96                 boolean result = true;
  97                 Abcd206ImportConfigurator config = state.getConfig();
  98                 //AbcdIO test = new AbcdIO();
  99                 String sourceName = config.getSource();
 100                 NodeList unitsList = getUnitsNodeList(sourceName);
 101                 if (unitsList != null){
 102                         logger.info("nb units to insert: "+unitsList.getLength());
 103
 104                         Abcd206DataHolder dataHolder = new Abcd206DataHolder();
 105
 106                         for (int i=0 ; i<unitsList.getLength() ; i++){
 107                                 this.setUnitPropertiesXML((Element)unitsList.item(i), dataHolder);
 108                                 result &= this.handleSingleUnit(config, dataHolder);
 109
 110                                 //compare the ABCD elements added in to the CDM and the unhandled ABCD elements
 111                                 compareABCDtoCDM(sourceName, dataHolder.knownABCDelements, dataHolder);
 112
 113                                 //reset the ABCD elements added in CDM
 114                                 //knownABCDelements = new ArrayList<String>();
 115                                 dataHolder.allABCDelements = new HashMap<String,String>();
 116                         }
 117                 }
 118
 119                 return result;
 120
 121         }
 122
 123         /*
 124          * Store the unit with its Gathering informations in the CDM
 125          */
 126         private boolean handleSingleUnit(Abcd206ImportConfigurator config, Abcd206DataHolder dataHolder){
 127                 boolean result = true;
 128
 129                 TransactionStatus tx = startTransaction();
 130                 try {
 131 //                      ReferenceBase sec = Database.NewInstance();
 132 //                      sec.setTitleCache("XML DATA");
 133                         ReferenceBase sec = config.getTaxonReference();
 134
 135                         //create facade
 136                         DerivedUnitFacade derivedUnitFacade = getFacade(dataHolder);
 137
 138
 139                         //handle identifications
 140                         handleIdentifications(config, derivedUnitFacade, sec, dataHolder);
 141
 142                         //handle collection data
 143                         setCollectionData(config, dataHolder, derivedUnitFacade);
 144
 145                         /**
 146                          * GATHERING EVENT
 147                          */
 148
 149                         //gathering event
 150                         UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(
 151                                         getTermService(), dataHolder.locality, dataHolder.languageIso, dataHolder.longitude,
 152                                         dataHolder.latitude, dataHolder.gatheringAgentList);
 153
 154                         //country
 155                         UnitsGatheringArea unitsGatheringArea =
 156                                 new UnitsGatheringArea(dataHolder.isocountry, dataHolder.country, getOccurrenceService());
 157                         NamedArea areaCountry = unitsGatheringArea.getArea();
 158
 159                         //other areas
 160                         unitsGatheringArea = new UnitsGatheringArea(dataHolder.namedAreaList);
 161                         ArrayList<NamedArea> nas = unitsGatheringArea.getAreas();
 162                         for (NamedArea namedArea : nas){
 163                                 unitsGatheringEvent.addArea(namedArea);
 164                         }
 165
 166                         //copy gathering event to facade
 167                         GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 168                         derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 169                         derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 170                         derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 171                         derivedUnitFacade.addCollectingArea(areaCountry);
 172                         //FIXME setCountry
 173                         derivedUnitFacade.addCollectingArea(areaCountry);
 174                         derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 175
 176                         //TODO exsiccatum
 177
 178
 179                         //add fieldNumber
 180                         derivedUnitFacade.setFieldNumber(dataHolder.fieldNumber);
 181
 182                         //join gatheringEvent to fieldObservation
 183
 184 //                      //add Multimedia URLs
 185                         if(dataHolder.multimediaObjects.size() > 0){
 186                                 MediaRepresentation representation;
 187                                 Media media;
 188                                 ImageMetaData imd ;
 189                                 URL url ;
 190                                 ImageFile imf;
 191                                 for (String multimediaObject : dataHolder.multimediaObjects){
 192                                         if( multimediaObject != null){
 193                                                 imd = ImageMetaData.newInstance();
 194                                                 url = new URL(multimediaObject);
 195                                                 imd.readMetaData(url.toURI(), 0);
 196                                                 //TODO ??
 197                                                 if (imd != null){
 198                                                         representation = MediaRepresentation.NewInstance();
 199                                                         imf = ImageFile.NewInstance(multimediaObject, null, imd);
 200                                                         representation.addRepresentationPart(imf);
 201                                                         media = Media.NewInstance();
 202                                                         media.addRepresentation(representation);
 203
 204                                                         derivedUnitFacade.addFieldObjectMedia(media);
 205                                                 }
 206                                         }
 207                                 }
 208                         }
 209
 210                         /**
 211                          * SAVE AND STORE DATA
 212                          */
 213                         getTermService().save(areaCountry);//TODO save area sooner
 214                         for (NamedArea area : nas){
 215                                 getTermService().save(area);//save it sooner (foreach area)
 216                         }
 217                         getTermService().saveLanguageData(unitsGatheringEvent.getLocality());//TODO needs to be saved ?? save it sooner
 218
 219                         getOccurrenceService().save(derivedUnitFacade.getDerivedUnit());
 220                         logger.info("saved ABCD specimen ...");
 221
 222
 223                 } catch (Exception e) {
 224                         logger.warn("Error when reading record!!");
 225                         e.printStackTrace();
 226                         result = false;
 227                 }
 228                 commitTransaction(tx);
 229
 230                 return result;
 231         }
 232
 233
 234         private void setCollectionData(Abcd206ImportConfigurator config,
 235                         Abcd206DataHolder dataHolder, DerivedUnitFacade derivedUnitFacade) {
 236                 //set catalogue number (unitID)
 237                 derivedUnitFacade.setCatalogNumber(dataHolder.unitID);
 238                 derivedUnitFacade.setAccessionNumber(dataHolder.accessionNumber);
 239                 derivedUnitFacade.setCollectorsNumber(dataHolder.collectorsNumber);
 240
 241
 242                 /**
 243                  * INSTITUTION & COLLECTION
 244                  */
 245                 //manage institution
 246                 Institution institution = this.getInstitution(dataHolder.institutionCode, config, dataHolder);
 247                 //manage collection
 248                 Collection collection = this.getCollection(dataHolder.collectionCode, institution, config, dataHolder);
 249                 //link specimen & collection
 250                 derivedUnitFacade.setCollection(collection);
 251         }
 252
 253
 254         private DerivedUnitFacade getFacade(Abcd206DataHolder dataHolder) {
 255                 /**
 256                  * SPECIMEN OR OBSERVATION OR LIVING
 257                  */
 258 //                      DerivedUnitBase derivedThing = null;
 259                 DerivedUnitType type = null;
 260
 261                 //create specimen
 262                 if (dataHolder.recordBasis != null){
 263                         if (dataHolder.recordBasis.toLowerCase().startsWith("s")) {//specimen
 264                                 type = DerivedUnitType.Specimen;
 265                         }else if (dataHolder.recordBasis.toLowerCase().startsWith("o")) {//observation
 266                                 type = DerivedUnitType.Observation;
 267                         }else if (dataHolder.recordBasis.toLowerCase().startsWith("l")) {//living -> fossil, herbarium sheet....???
 268                                 type = DerivedUnitType.LivingBeing;
 269                         }
 270                         if (type == null){
 271                                 logger.info("The basis of record does not seem to be known: " + dataHolder.recordBasis);
 272                                 type = DerivedUnitType.DerivedUnit;
 273                         }
 274                         //TODO fossils?
 275                 }else{
 276                         logger.info("The basis of record is null");
 277                         type = DerivedUnitType.DerivedUnit;
 278                 }
 279                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 280                 return derivedUnitFacade;
 281         }
 282
 283
 284         /*
 285          * Return the list of root nodes for an ABCD 2.06 XML file
 286          * @param fileName: the file's location
 287          * @return the list of root nodes ("Unit")
 288          */
 289         private static NodeList getUnitsNodeList(String urlFileName){
 290                 NodeList unitList = null;
 291                 try {
 292                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 293                         DocumentBuilder builder = factory.newDocumentBuilder();
 294                         URL url = new URL(urlFileName);
 295                         Object o = url.getContent();
 296                         InputStream is = (InputStream)o;
 297                         Document document = builder.parse(is);
 298                         Element root = document.getDocumentElement();
 299                         unitList = root.getElementsByTagName("Unit");
 300
 301                 }catch(Exception e){
 302                         logger.warn(e);
 303                 }
 304                 return unitList;
 305         }
 306
 307
 308         /*
 309          * Store the unit's properties into variables
 310          * Look which unit is the preferred one
 311          * Look what kind of name it is supposed to be, for the parsing (Botanical, Zoological)
 312          * @param racine: the root node for a single unit
 313          */
 314         private void setUnitPropertiesXML(Element root, Abcd206DataHolder dataHolder){
 315                 try{
 316                         NodeList group;
 317
 318 //                      try{afficherInfos(racine, 0);}catch (Exception e) {logger.info(e);}
 319                         group = root.getChildNodes();
 320 //                      logger.info("ABCD ELEMENT not stored: "+group.item(i).getNodeName().toString()+" - value: "+group.item(i).getTextContent());
 321                         for (int i = 0; i < group.getLength(); i++){
 322                                 if (group.item(i).getNodeName().equals("Identifications")){
 323                                         group = group.item(i).getChildNodes();
 324                                         break;
 325                                 }
 326                         }
 327                         dataHolder.identificationList = new ArrayList<String>();
 328                         dataHolder.atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
 329                         dataHolder.referenceList = new ArrayList<String>();
 330                         dataHolder.multimediaObjects = new ArrayList<String>();
 331
 332                         this.getScientificNames(group, dataHolder);
 333
 334 //                      logger.info("this.identificationList "+this.identificationList.toString());
 335                         this.getIDs(root, dataHolder);
 336                         this.getRecordBasis(root, dataHolder);
 337                         this.getMultimedia(root, dataHolder);
 338                         this.getNumbers(root, dataHolder);
 339                         this.getGeolocation(root, dataHolder);
 340                         this.getGatheringPeople(root, dataHolder);
 341
 342                 } catch (Exception e) {
 343                         logger.info("Error occured while parsing XML file"+e);
 344                 }
 345         }
 346
 347         String path= "";
 348         private void getHierarchie(Node node){
 349                 while (node != null && node.getNodeName() != "DataSets"){
 350 //                      logger.info(node.getParentNode().getNodeName());
 351                         path = node.getParentNode().getNodeName()+"/"+path;
 352                         node = node.getParentNode();
 353                 }
 354 //              logger.info("path: "+path);
 355         }
 356
 357         private void getScientificNames(NodeList group, Abcd206DataHolder dataHolder){
 358                 NodeList identifications,results;
 359                 String tmpName = null;
 360                 for (int j=0; j< group.getLength(); j++){
 361                         if(group.item(j).getNodeName().equals("Identification")){
 362                                 dataHolder.nomenclatureCode ="";
 363                                 identifications = group.item(j).getChildNodes();
 364                                 for (int m=0; m<identifications.getLength();m++){
 365                                         if(identifications.item(m).getNodeName().equals("Result")){
 366                                                 results = identifications.item(m).getChildNodes();
 367                                                 for(int k=0; k<results.getLength();k++){
 368                                                         if (results.item(k).getNodeName().equals("TaxonIdentified")){
 369                                                                 tmpName=this.getScientificName(results.item(k), dataHolder);
 370                                                         }
 371                                                 }
 372                                         }else if(identifications.item(m).getNodeName().equals("PreferredFlag")){
 373                                                 if (dataHolder.nomenclatureCode != null && dataHolder.nomenclatureCode !=""){
 374                                                         dataHolder.identificationList.add(tmpName+"_preferred_"+identifications.item(m).getTextContent()+"_code_" + dataHolder.nomenclatureCode);
 375                                                 }else{
 376                                                         dataHolder.identificationList.add(tmpName+"_preferred_"+identifications.item(m).getTextContent());
 377                                                 }
 378                                                 path=identifications.item(m).getNodeName();
 379                                                 getHierarchie(identifications.item(m));
 380                                                 dataHolder.knownABCDelements.add(path);
 381                                                 path="";
 382                                         }else if (identifications.item(m).getNodeName().equals("References")){
 383                                                 this.getReferences(identifications.item(m), dataHolder);
 384                                         }
 385                                 }
 386                         }
 387                 }
 388                 boolean hasPref=false;
 389                 for (int j=0; j< group.getLength(); j++){
 390                         if(group.item(j).getNodeName().equals("Identification")){
 391                                 dataHolder.nomenclatureCode ="";
 392                                 identifications = group.item(j).getChildNodes();
 393                                 for (int m=0; m<identifications.getLength();m++){
 394                                         if(identifications.item(m).getNodeName().equals("Result")){
 395                                                 results = identifications.item(m).getChildNodes();
 396                                                 for(int k=0; k<results.getLength();k++){
 397                                                         if (results.item(k).getNodeName().equals("TaxonIdentified")){
 398                                                                 tmpName=this.getScientificName(results.item(k), dataHolder);
 399                                                         }
 400                                                 }
 401                                         }
 402                                         if(identifications.item(m).getNodeName().equals("PreferredFlag")){
 403                                                 hasPref=true;
 404                                         }
 405                                 }
 406                                 if ( !hasPref && tmpName != null){
 407                                         if (dataHolder.nomenclatureCode != null && dataHolder.nomenclatureCode !=""){
 408                                                 dataHolder.identificationList.add(tmpName+"_preferred_"+"0"+"_code_" + dataHolder.nomenclatureCode);
 409                                         } else {
 410                                                 dataHolder.identificationList.add(tmpName+"_preferred_"+"0");
 411                                         }
 412                                 }
 413                         }
 414                 }
 415         }
 416
 417
 418
 419         private void getReferences(Node result, Abcd206DataHolder dataHolder){
 420                 NodeList results,reference;
 421                 results = result.getChildNodes();
 422                 for(int k=0; k<results.getLength();k++){
 423                         if (results.item(k).getNodeName().equals("Reference")){
 424                                 reference = results.item(k).getChildNodes();
 425                                 for(int l=0;l<reference.getLength();l++){
 426                                         if (reference.item(l).getNodeName().equals("TitleCitation")){
 427                                                 path = reference.item(l).getNodeName();
 428                                                 dataHolder.referenceList.add(reference.item(l).getTextContent());
 429                                                 getHierarchie(reference.item(l));
 430                                                 dataHolder.knownABCDelements.add(path);
 431                                                 path="";
 432                                         }
 433                                 }
 434                         }
 435                 }
 436         }
 437
 438         private String getScientificName(Node result, Abcd206DataHolder dataHolder){
 439                 NodeList taxonsIdentified, scnames, atomised;
 440                 String tmpName = "";
 441                 dataHolder.atomisedStr = "";
 442                 taxonsIdentified = result.getChildNodes();
 443                 for (int l=0; l<taxonsIdentified.getLength(); l++){
 444                         if (taxonsIdentified.item(l).getNodeName().equals("ScientificName")){
 445                                 scnames = taxonsIdentified.item(l).getChildNodes();
 446                                 for (int n=0;n<scnames.getLength();n++){
 447                                         if (scnames.item(n).getNodeName().equals("FullScientificNameString")){
 448                                                 path=scnames.item(n).getNodeName();
 449                                                 tmpName = scnames.item(n).getTextContent();
 450                                                 getHierarchie(scnames.item(n));
 451                                                 dataHolder.knownABCDelements.add(path);
 452                                                 path="";
 453                                         }
 454                                         if (scnames.item(n).getNodeName().equals("NameAtomised")){
 455                                                 try {
 456                                                         if (scnames.item(n).hasChildNodes()){
 457                                                                 dataHolder.nomenclatureCode = scnames.item(n).getChildNodes().item(1).getNodeName();
 458                                                         }
 459                                                 } catch (Exception e) {
 460                                                         dataHolder.nomenclatureCode ="";
 461                                                 }
 462                                                 atomised = scnames.item(n).getChildNodes().item(1).getChildNodes();
 463                                                 dataHolder.atomisedIdentificationList.add(this.getAtomisedNames(dataHolder.nomenclatureCode, atomised, dataHolder));
 464                                         }
 465                                 }
 466                         }
 467                 }
 468                 return tmpName;
 469         }
 470
 471         private HashMap<String,String> getAtomisedNames(String code, NodeList atomised, Abcd206DataHolder dataHolder){
 472                 if (code.equals("Botanical")){
 473                         return this.getAtomisedBotanical(atomised, dataHolder);
 474                 }
 475                 if (code.equals("Bacterial")){
 476                         return this.getAtomisedBacterial(atomised, dataHolder);
 477                 }
 478                 if (code.equals("NameViral")){
 479                         return this.getAtomisedViral(atomised, dataHolder);
 480                 }
 481                 if (code.equals("NameZoological")){
 482                         return this.getAtomisedZoological(atomised,dataHolder);
 483                 }
 484                 return new HashMap<String,String>();
 485         }
 486
 487         private HashMap<String,String> getAtomisedZoological(NodeList atomised, Abcd206DataHolder dataHolder){
 488                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 489                 for (int i=0;i<atomised.getLength();i++){
 490                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 491                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 492                                 path=atomised.item(i).getNodeName();
 493                                 getHierarchie(atomised.item(i));
 494                                 dataHolder.knownABCDelements.add(path);
 495                                 path="";
 496                         }
 497                         if(atomised.item(i).getNodeName().equals("Subgenus")){
 498                                 atomisedMap.put("Subgenus",atomised.item(i).getTextContent());
 499                                 path=atomised.item(i).getNodeName();
 500                                 getHierarchie(atomised.item(i));
 501                                 dataHolder.knownABCDelements.add(path);
 502                                 path="";
 503                         }
 504                         if(atomised.item(i).getNodeName().equals("SpeciesEpithet")){
 505                                 atomisedMap.put("SpeciesEpithet",atomised.item(i).getTextContent());
 506                                 path=atomised.item(i).getNodeName();
 507                                 getHierarchie(atomised.item(i));
 508                                 dataHolder.knownABCDelements.add(path);
 509                                 path="";
 510                         }
 511                         if(atomised.item(i).getNodeName().equals("SubspeciesEpithet")){
 512                                 atomisedMap.put("SubspeciesEpithet",atomised.item(i).getTextContent());
 513                                 path=atomised.item(i).getNodeName();
 514                                 getHierarchie(atomised.item(i));
 515                                 dataHolder.knownABCDelements.add(path);
 516                                 path="";
 517                         }
 518                         if(atomised.item(i).getNodeName().equals("AuthorTeamOriginalAndYear")){
 519                                 atomisedMap.put("AuthorTeamOriginalAndYear",atomised.item(i).getTextContent());
 520                                 path=atomised.item(i).getNodeName();
 521                                 getHierarchie(atomised.item(i));
 522                                 dataHolder.knownABCDelements.add(path);
 523                                 path="";
 524                         }
 525                         if(atomised.item(i).getNodeName().equals("AuthorTeamParenthesisAndYear")){
 526                                 atomisedMap.put("AuthorTeamParenthesisAndYear",atomised.item(i).getTextContent());
 527                                 path=atomised.item(i).getNodeName();
 528                                 getHierarchie(atomised.item(i));
 529                                 dataHolder.knownABCDelements.add(path);
 530                                 path="";
 531                         }
 532                         if(atomised.item(i).getNodeName().equals("CombinationAuthorTeamAndYear")){
 533                                 atomisedMap.put("CombinationAuthorTeamAndYear",atomised.item(i).getTextContent());
 534                                 path=atomised.item(i).getNodeName();
 535                                 getHierarchie(atomised.item(i));
 536                                 dataHolder.knownABCDelements.add(path);
 537                                 path="";
 538                         }
 539                         if(atomised.item(i).getNodeName().equals("Breed")){
 540                                 atomisedMap.put("Breed",atomised.item(i).getTextContent());
 541                                 path=atomised.item(i).getNodeName();
 542                                 getHierarchie(atomised.item(i));
 543                                 dataHolder.knownABCDelements.add(path);
 544                                 path="";
 545                         }
 546                         if(atomised.item(i).getNodeName().equals("NamedIndividual")){
 547                                 atomisedMap.put("NamedIndividual",atomised.item(i).getTextContent());
 548                                 path=atomised.item(i).getNodeName();
 549                                 getHierarchie(atomised.item(i));
 550                                 dataHolder.knownABCDelements.add(path);
 551                                 path="";
 552                         }
 553                 }
 554                 return atomisedMap;
 555         }
 556
 557         private HashMap<String,String> getAtomisedViral(NodeList atomised, Abcd206DataHolder dataHolder){
 558                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 559                 for (int i=0;i<atomised.getLength();i++){
 560                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 561                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 562                                 path=atomised.item(i).getNodeName();
 563                                 getHierarchie(atomised.item(i));
 564                                 dataHolder.knownABCDelements.add(path);
 565                                 path="";
 566                         }
 567                         if(atomised.item(i).getNodeName().equals("ViralSpeciesDesignation")){
 568                                 atomisedMap.put("ViralSpeciesDesignation", atomised.item(i).getTextContent());
 569                                 path=atomised.item(i).getNodeName();
 570                                 getHierarchie(atomised.item(i));
 571                                 dataHolder.knownABCDelements.add(path);
 572                                 path="";
 573                         }
 574                         if(atomised.item(i).getNodeName().equals("Acronym")){
 575                                 atomisedMap.put("Acronym",atomised.item(i).getTextContent());
 576                                 path=atomised.item(i).getNodeName();
 577                                 getHierarchie(atomised.item(i));
 578                                 dataHolder.knownABCDelements.add(path);
 579                                 path="";
 580                         }
 581                 }
 582                 return atomisedMap;
 583         }
 584
 585         private HashMap<String,String> getAtomisedBotanical(NodeList atomised, Abcd206DataHolder dataHolder){
 586                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 587                 for (int i=0;i<atomised.getLength();i++){
 588                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 589                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 590                                 path=atomised.item(i).getNodeName();
 591                                 getHierarchie(atomised.item(i));
 592                                 dataHolder.knownABCDelements.add(path);
 593                                 path="";
 594                         }
 595                         if(atomised.item(i).getNodeName().equals("FirstEpithet")){
 596                                 atomisedMap.put("FirstEpithet",atomised.item(i).getTextContent());
 597                                 path=atomised.item(i).getNodeName();
 598                                 getHierarchie(atomised.item(i));
 599                                 dataHolder.knownABCDelements.add(path);
 600                                 path="";
 601                         }
 602                         if(atomised.item(i).getNodeName().equals("InfraspecificEpithet")){
 603                                 atomisedMap.put("InfraSpeEpithet", atomised.item(i).getTextContent());
 604                                 path=atomised.item(i).getNodeName();
 605                                 getHierarchie(atomised.item(i));
 606                                 dataHolder.knownABCDelements.add(path);
 607                                 path="";
 608                         }
 609                         if(atomised.item(i).getNodeName().equals("Rank")){
 610                                 atomisedMap.put("Rank",atomised.item(i).getTextContent());
 611                                 path=atomised.item(i).getNodeName();
 612                                 getHierarchie(atomised.item(i));
 613                                 dataHolder.knownABCDelements.add(path);
 614                                 path="";
 615                         }
 616                         if(atomised.item(i).getNodeName().equals("HybridFlag")){
 617                                 atomisedMap.put("HybridFlag",atomised.item(i).getTextContent());
 618                                 path=atomised.item(i).getNodeName();
 619                                 getHierarchie(atomised.item(i));
 620                                 dataHolder.knownABCDelements.add(path);
 621                                 path="";
 622                         }
 623                         if(atomised.item(i).getNodeName().equals("AuthorTeamParenthesis")){
 624                                 atomisedMap.put("AuthorTeamParenthesis",atomised.item(i).getTextContent());
 625                                 path=atomised.item(i).getNodeName();
 626                                 getHierarchie(atomised.item(i));
 627                                 dataHolder.knownABCDelements.add(path);
 628                                 path="";
 629                         }
 630                         if(atomised.item(i).getNodeName().equals("AuthorTeam")){
 631                                 atomisedMap.put("AuthorTeam",atomised.item(i).getTextContent());
 632                                 path=atomised.item(i).getNodeName();
 633                                 getHierarchie(atomised.item(i));
 634                                 dataHolder.knownABCDelements.add(path);
 635                                 path="";
 636                         }
 637                         if(atomised.item(i).getNodeName().equals("CultivarGroupName")){
 638                                 atomisedMap.put("CultivarGroupName",atomised.item(i).getTextContent());
 639                                 path=atomised.item(i).getNodeName();
 640                                 getHierarchie(atomised.item(i));
 641                                 dataHolder.knownABCDelements.add(path);
 642                                 path="";
 643                         }
 644                         if(atomised.item(i).getNodeName().equals("CultivarName")){
 645                                 atomisedMap.put("CultivarName",atomised.item(i).getTextContent());
 646                                 path=atomised.item(i).getNodeName();
 647                                 getHierarchie(atomised.item(i));
 648                                 dataHolder.knownABCDelements.add(path);
 649                                 path="";
 650                         }
 651                         if(atomised.item(i).getNodeName().equals("TradeDesignationNames")){
 652                                 atomisedMap.put("Trade",atomised.item(i).getTextContent());
 653                                 path=atomised.item(i).getNodeName();
 654                                 getHierarchie(atomised.item(i));
 655                                 dataHolder.knownABCDelements.add(path);
 656                                 path="";
 657                         }
 658                 }
 659                 return atomisedMap;
 660         }
 661
 662         private HashMap<String,String> getAtomisedBacterial(NodeList atomised, Abcd206DataHolder dataHolder){
 663                 HashMap<String,String> atomisedMap = new HashMap<String,String>();
 664                 for (int i=0;i<atomised.getLength();i++){
 665                         if(atomised.item(i).getNodeName().equals("GenusOrMonomial")){
 666                                 atomisedMap.put("Genus",atomised.item(i).getTextContent());
 667                                 path=atomised.item(i).getNodeName();
 668                                 getHierarchie(atomised.item(i));
 669                                 dataHolder.knownABCDelements.add(path);
 670                                 path="";
 671                         }
 672                         if(atomised.item(i).getNodeName().equals("Subgenus")){
 673                                 atomisedMap.put("SubGenus",atomised.item(i).getTextContent());
 674                                 path=atomised.item(i).getNodeName();
 675                                 getHierarchie(atomised.item(i));
 676                                 dataHolder.knownABCDelements.add(path);
 677                                 path="";
 678                         }
 679                         if(atomised.item(i).getNodeName().equals("SubgenusAuthorAndYear")){
 680                                 atomisedMap.put("SubgenusAuthorAndYear",atomised.item(i).getTextContent());
 681                                 path=atomised.item(i).getNodeName();
 682                                 getHierarchie(atomised.item(i));
 683                                 dataHolder.knownABCDelements.add(path);
 684                                 path="";
 685                         }
 686                         if(atomised.item(i).getNodeName().equals("SpeciesEpithet")){
 687                                 atomisedMap.put("SpeciesEpithet",atomised.item(i).getTextContent());
 688                                 path=atomised.item(i).getNodeName();
 689                                 getHierarchie(atomised.item(i));
 690                                 dataHolder.knownABCDelements.add(path);
 691                                 path="";
 692                         }
 693                         if(atomised.item(i).getNodeName().equals("SubspeciesEpithet")){
 694                                 atomisedMap.put("SubspeciesEpithet",atomised.item(i).getTextContent());
 695                                 path=atomised.item(i).getNodeName();
 696                                 getHierarchie(atomised.item(i));
 697                                 dataHolder.knownABCDelements.add(path);
 698                                 path="";
 699                         }
 700                         if(atomised.item(i).getNodeName().equals("ParentheticalAuthorTeamAndYear")){
 701                                 atomisedMap.put("ParentheticalAuthorTeamAndYear",atomised.item(i).getTextContent());
 702                                 path=atomised.item(i).getNodeName();
 703                                 getHierarchie(atomised.item(i));
 704                                 dataHolder.knownABCDelements.add(path);
 705                                 path="";
 706                         }
 707                         if(atomised.item(i).getNodeName().equals("AuthorTeamAndYear")){
 708                                 atomisedMap.put("AuthorTeamAndYear",atomised.item(i).getTextContent());
 709                                 path=atomised.item(i).getNodeName();
 710                                 getHierarchie(atomised.item(i));
 711                                 dataHolder.knownABCDelements.add(path);
 712                                 path="";
 713                         }
 714                         if(atomised.item(i).getNodeName().equals("NameApprobation")){
 715                                 atomisedMap.put("NameApprobation",atomised.item(i).getTextContent());
 716                                 path=atomised.item(i).getNodeName();
 717                                 getHierarchie(atomised.item(i));
 718                                 dataHolder.knownABCDelements.add(path);
 719                                 path="";
 720                         }
 721                 }
 722                 return atomisedMap;
 723         }
 724
 725         private void getIDs(Element root, Abcd206DataHolder dataHolder){
 726                 NodeList group;
 727                 try {
 728                         group = root.getElementsByTagName("SourceInstitutionID");
 729                         path=group.item(0).getNodeName();
 730                         getHierarchie(group.item(0));
 731                         dataHolder.knownABCDelements.add(path);
 732                         path="";
 733                         dataHolder.institutionCode = group.item(0).getTextContent();
 734                 } catch (NullPointerException e) {
 735                         dataHolder.institutionCode= "";
 736                 }
 737                 try {
 738                         group = root.getElementsByTagName("SourceID");
 739                         path=group.item(0).getNodeName();
 740                         getHierarchie(group.item(0));
 741                         dataHolder.knownABCDelements.add(path);
 742                         path="";
 743                         dataHolder.collectionCode = group.item(0).getTextContent();
 744                 } catch (NullPointerException e) {
 745                         dataHolder.collectionCode = "";
 746                 }
 747                 try {
 748                         group = root.getElementsByTagName("UnitID");
 749                         path=group.item(0).getNodeName();
 750                         getHierarchie(group.item(0));
 751                         dataHolder.knownABCDelements.add(path);
 752                         path="";
 753                         dataHolder.unitID = group.item(0).getTextContent();
 754                 } catch (NullPointerException e) {
 755                         dataHolder.unitID = "";
 756                 }
 757         }
 758
 759         private void getRecordBasis(Element root, Abcd206DataHolder dataHolder){
 760                 NodeList group;
 761                 try {
 762                         group = root.getElementsByTagName("RecordBasis");
 763                         path=group.item(0).getNodeName();
 764                         getHierarchie(group.item(0));
 765                         dataHolder.knownABCDelements.add(path);
 766                         path="";
 767                         dataHolder.recordBasis = group.item(0).getTextContent();
 768                 } catch (NullPointerException e) {
 769                         dataHolder.recordBasis = "";
 770                 }
 771         }
 772
 773         private void getMultimedia(Element root, Abcd206DataHolder dataHolder){
 774                 NodeList group, multimedias, multimedia;
 775                 try {
 776                         group = root.getElementsByTagName("MultiMediaObjects");
 777                         for(int i=0;i<group.getLength();i++){
 778                                 multimedias = group.item(i).getChildNodes();
 779                                 for (int j=0;j<multimedias.getLength();j++){
 780                                         if (multimedias.item(j).getNodeName().equals("MultiMediaObject")){
 781                                                 multimedia = multimedias.item(j).getChildNodes();
 782                                                 for (int k=0;k<multimedia.getLength();k++){
 783                                                         if(multimedia.item(k).getNodeName().equals("FileURI")){
 784                                                                 dataHolder.multimediaObjects.add(multimedia.item(k).getTextContent());
 785                                                                 path = multimedia.item(k).getNodeName();
 786                                                                 getHierarchie(multimedia.item(k));
 787                                                                 dataHolder.knownABCDelements.add(path);
 788                                                                 path="";
 789                                                         }
 790                                                 }
 791                                         }
 792                                 }
 793                         }
 794                 } catch (NullPointerException e) {
 795                         logger.info(e);
 796                 }
 797         }
 798
 799         private void getNumbers(Element root, Abcd206DataHolder dataHolder){
 800                 NodeList group;
 801                 try {
 802                         group = root.getElementsByTagName("AccessionNumber");
 803                         path=group.item(0).getNodeName();
 804                         getHierarchie(group.item(0));
 805                         dataHolder.knownABCDelements.add(path);
 806                         path="";
 807                         dataHolder.accessionNumber = group.item(0).getTextContent();
 808                 } catch (NullPointerException e) {
 809                         dataHolder.accessionNumber = "";
 810                 }
 811                 try {
 812                         group = root.getElementsByTagName("CollectorsFieldNumber");
 813                         path=group.item(0).getNodeName();
 814                         getHierarchie(group.item(0));
 815                         dataHolder.knownABCDelements.add(path);
 816                         path="";
 817                         dataHolder.fieldNumber = group.item(0).getTextContent();
 818                 } catch (NullPointerException e) {
 819                         dataHolder.fieldNumber = "";
 820                 }
 821
 822                 try {
 823                         group = root.getElementsByTagName("CollectorsNumber");
 824                         path=group.item(0).getNodeName();
 825                         getHierarchie(group.item(0));
 826                         dataHolder.knownABCDelements.add(path);
 827                         path="";
 828                         dataHolder.collectorsNumber = group.item(0).getTextContent();
 829                 } catch (NullPointerException e) {
 830                         dataHolder.collectorsNumber = "";
 831                 }
 832
 833                 try {
 834                         group = root.getElementsByTagName("AccessionNumber");
 835                         path=group.item(0).getNodeName();
 836                         getHierarchie(group.item(0));
 837                         dataHolder.knownABCDelements.add(path);
 838                         path="";
 839                         dataHolder.accessionNumber = group.item(0).getTextContent();
 840                 } catch (NullPointerException e) {
 841                         dataHolder.accessionNumber = "";
 842                 }
 843         }
 844
 845         private void getGeolocation(Element root, Abcd206DataHolder dataHolder){
 846                 NodeList group, childs;
 847                 try {
 848                         group = root.getElementsByTagName("LocalityText");
 849                         path=group.item(0).getNodeName();
 850                         getHierarchie(group.item(0));
 851                         dataHolder.knownABCDelements.add(path);
 852                         path="";
 853                         dataHolder.locality = group.item(0).getTextContent();
 854                         if (group.item(0).hasAttributes())
 855                                 if (group.item(0).getAttributes().getNamedItem("lang") != null)
 856                                         dataHolder.languageIso = group.item(0).getAttributes().getNamedItem("lang").getTextContent();
 857                 } catch (NullPointerException e) {
 858                         dataHolder.locality = "";
 859                 }
 860                 try {
 861                         group = root.getElementsByTagName("LongitudeDecimal");
 862                         path=group.item(0).getNodeName();
 863                         getHierarchie(group.item(0));
 864                         dataHolder.knownABCDelements.add(path);
 865                         path="";
 866                         dataHolder.longitude = Double.valueOf(group.item(0).getTextContent());
 867                 } catch (NullPointerException e) {
 868                         dataHolder.longitude=0.0;
 869                 }
 870                 try {
 871                         group = root.getElementsByTagName("LatitudeDecimal");
 872                         path=group.item(0).getNodeName();
 873                         getHierarchie(group.item(0));
 874                         dataHolder.knownABCDelements.add(path);
 875                         path="";
 876                         dataHolder.latitude = Double.valueOf(group.item(0).getTextContent());
 877                 } catch (NullPointerException e) {
 878                         dataHolder.latitude=0.0;
 879                 }
 880                 try {
 881                         group = root.getElementsByTagName("Country");
 882                         childs = group.item(0).getChildNodes();
 883                         for (int i=0;i<childs.getLength(); i++){
 884                                 if(childs.item(i).getNodeName() == "Name"){
 885                                         path=childs.item(i).getNodeName();
 886                                         getHierarchie(childs.item(i));
 887                                         dataHolder.knownABCDelements.add(path);
 888                                         path="";
 889                                         dataHolder.country = childs.item(i).getTextContent();
 890                                 }
 891                         }
 892                 } catch (NullPointerException e) {
 893                         dataHolder.country = "";
 894                 }
 895                 try {
 896                         group = root.getElementsByTagName("Country");
 897                         childs = group.item(0).getChildNodes();
 898                         for (int i=0;i<childs.getLength(); i++){
 899                                 if(childs.item(i).getNodeName() == "ISO3166Code"){
 900                                         path=childs.item(i).getNodeName();
 901                                         getHierarchie(childs.item(i));
 902                                         dataHolder.knownABCDelements.add(path);
 903                                         path="";
 904                                         dataHolder.isocountry = childs.item(i).getTextContent();
 905                                 }
 906                         }
 907                 } catch (NullPointerException e) {
 908                         dataHolder.isocountry = "";
 909                 }
 910                 try {
 911                         group = root.getElementsByTagName("Altitude");
 912                         for (int i=0;i<group.getLength();i++){
 913                                 childs = group.item(i).getChildNodes();
 914                                 for (int j=0;j<childs.getLength();j++){
 915                                         if (childs.item(j).getNodeName().equals("MeasurementOrFactText")){
 916                                                 path=childs.item(j).getNodeName();
 917                                                 getHierarchie(childs.item(j));
 918                                                 dataHolder.knownABCDelements.add(path);
 919                                                 path="";
 920                                                 dataHolder.altitude = Integer.valueOf(childs.item(j).getTextContent());
 921                                         }
 922                                 }
 923                         }
 924                 } catch (NullPointerException e) {
 925                         dataHolder.altitude = -9999;
 926                 }
 927
 928                 try {
 929                         group = root.getElementsByTagName("Depth");
 930                         path=group.item(0).getNodeName();
 931                         getHierarchie(group.item(0));
 932                         dataHolder.knownABCDelements.add(path);
 933                         path="";
 934                         dataHolder.depth = Integer.valueOf(group.item(0).getTextContent());
 935                 } catch (NullPointerException e) {
 936                         dataHolder.depth = -9999;
 937                 }
 938
 939                 try{
 940                         group = root.getElementsByTagName("NamedArea");
 941                         dataHolder.namedAreaList = new ArrayList<String>();
 942                         for (int i=0;i<group.getLength();i++){
 943                                 childs = group.item(i).getChildNodes();
 944                                 for (int j=0; j<childs.getLength();j++){
 945                                         if (childs.item(j).getNodeName().equals("AreaName")){
 946                                                 path = childs.item(j).getNodeName();
 947                                                 getHierarchie(childs.item(j));
 948                                                 dataHolder.knownABCDelements.add(path);
 949                                                 path="";
 950                                                 dataHolder.namedAreaList.add(childs.item(j).getTextContent());
 951                                         }
 952                                 }
 953                         }
 954                 }catch(NullPointerException e){
 955                         dataHolder.namedAreaList = new ArrayList<String>();
 956                 }
 957         }
 958
 959         private void getGatheringPeople(Element root, Abcd206DataHolder dataHolder){
 960                 NodeList group, childs, person;
 961                 try {
 962                         group = root.getElementsByTagName("GatheringAgent");
 963                         dataHolder.gatheringAgentList = new ArrayList<String>();
 964                         for (int i=0; i< group.getLength(); i++){
 965                                 childs = group.item(i).getChildNodes();
 966                                 for (int j=0; j<childs.getLength();j++){
 967                                         if (childs.item(j).getNodeName().equals("Person")){
 968                                                 person = childs.item(j).getChildNodes();
 969                                                 for (int k=0; k<person.getLength(); k++){
 970                                                         if (person.item(k).getNodeName().equals("FullName")){
 971                                                                 path=person.item(k).getNodeName();
 972                                                                 getHierarchie(person.item(k));
 973                                                                 dataHolder.knownABCDelements.add(path);
 974                                                                 path="";
 975                                                                 dataHolder.gatheringAgentList.add(person.item(k).getTextContent());
 976                                                         }
 977                                                 }
 978                                         }
 979
 980                                 }
 981                         }
 982                 } catch (NullPointerException e) {
 983                         dataHolder.gatheringAgentList = new ArrayList<String>();
 984                 }
 985         }
 986
 987         private Institution getInstitution(String institutionCode, Abcd206ImportConfigurator config, Abcd206DataHolder dataHolder){
 988                 Institution institution;
 989                 List<Institution> institutions;
 990                 try{
 991                         logger.info(dataHolder.institutionCode);
 992                         institutions = getAgentService().searchInstitutionByCode(dataHolder.institutionCode);
 993                 }catch(Exception e){
 994                         institutions=new ArrayList<Institution>();
 995                 }
 996                 if (institutions.size() ==0 || !config.getReUseExistingMetadata()){
 997                         logger.info("Institution (agent) unknown or not allowed to reuse existing metadata");
 998                         //create institution
 999                         institution = Institution.NewInstance();
1000                         institution.setCode(dataHolder.institutionCode);
1001                 }
1002                 else{
1003                         logger.info("Institution (agent) already in the db");
1004                         institution = institutions.get(0);
1005                 }
1006                 return institution;
1007         }
1008
1009         /*
1010          * Look if the Collection does already exists
1011          * @param collectionCode: a string
1012          * @param institution: the current Institution
1013          * @param app
1014          * @return the Collection (existing or new)
1015          */
1016         private Collection getCollection(String collectionCode, Institution institution, Abcd206ImportConfigurator config, Abcd206DataHolder dataHolder){
1017                 Collection collection = Collection.NewInstance();
1018                 List<Collection> collections;
1019                 try{
1020                         collections = getCollectionService().searchByCode(dataHolder.collectionCode);
1021                 }catch(Exception e){
1022                         collections=new ArrayList<Collection>();
1023                 }
1024                 if (collections.size() ==0 || !config.getReUseExistingMetadata()){
1025                         logger.info("Collection not found or do not reuse existing metadata  " + dataHolder.collectionCode);
1026                         //create new collection
1027                         collection.setCode(dataHolder.collectionCode);
1028                         collection.setCodeStandard("GBIF");
1029                         collection.setInstitute(institution);
1030                 }
1031                 else{
1032                         boolean collectionFound=false;
1033                         for (int i=0; i<collections.size(); i++){
1034                                 collection = collections.get(i);
1035                                 try {
1036                                         if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())){
1037                                                 //found a collection with the same code and the same institution
1038                                                 collectionFound=true;
1039                                         }
1040                                 } catch (NullPointerException e) {}
1041                         }
1042                         if (!collectionFound){
1043                                 collection.setCode(dataHolder.collectionCode);
1044                                 collection.setCodeStandard("GBIF");
1045                                 collection.setInstitute(institution);
1046                         }
1047
1048                 }
1049                 return collection;
1050         }
1051
1052         /*
1053          *
1054          * @param app
1055          * @param derivedThing
1056          * @param sec
1057          */
1058         private void handleIdentifications(Abcd206ImportConfigurator config, DerivedUnitFacade facade, ReferenceBase sec, Abcd206DataHolder dataHolder){
1059                 NonViralName<?> taxonName = null;
1060                 String fullScientificNameString;
1061                 Taxon taxon = null;
1062                 DeterminationEvent determinationEvent = null;
1063                 List<TaxonBase> names = null;
1064
1065                 String scientificName="";
1066                 boolean preferredFlag=false;
1067
1068                 for (int i = 0; i < dataHolder.identificationList.size(); i++) {
1069                         fullScientificNameString = dataHolder.identificationList.get(i);
1070                         fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
1071                         if (fullScientificNameString.indexOf("_preferred_") != -1){
1072                                 scientificName = fullScientificNameString.split("_preferred_")[0];
1073                                 String pTmp = fullScientificNameString.split("_preferred_")[1].split("_code_")[0];
1074                                 if (pTmp.equals("1") || pTmp.toLowerCase().indexOf("true") != -1){
1075                                         preferredFlag=true;
1076                                 } else {
1077                                         preferredFlag=false;
1078                                 }
1079                         }
1080                         else{
1081                                 scientificName = fullScientificNameString;
1082                         }
1083                         logger.info(fullScientificNameString);
1084                         if (fullScientificNameString.indexOf("_code_") != -1){
1085                                 dataHolder.nomenclatureCode = fullScientificNameString.split("_code_")[1];
1086                         }
1087                         if (config.getDoAutomaticParsing() || dataHolder.atomisedIdentificationList == null || dataHolder.atomisedIdentificationList.size()==0){
1088                                 taxonName = this.parseScientificName(scientificName, dataHolder);
1089                         } else {
1090                                 if (dataHolder.atomisedIdentificationList != null || dataHolder.atomisedIdentificationList.size()>0){
1091                                         taxonName = this.setTaxonNameByType(dataHolder.atomisedIdentificationList.get(i), scientificName, dataHolder);
1092                                 }
1093                         }
1094                         if(taxonName == null){
1095                                 taxonName = NonViralName.NewInstance(null);
1096                                 taxonName.setFullTitleCache(scientificName);
1097                         }
1098                         if (config.getDoReUseTaxon()){
1099                                 try{
1100                                         names = getTaxonService().searchTaxaByName(scientificName, sec);
1101                                         taxon = (Taxon)names.get(0);
1102                                 } catch(Exception e){
1103                                         taxon=null;
1104                                 }
1105                         }
1106 //                      taxonName = NonViralName.NewInstance(null);
1107 //                      taxonName.setFullTitleCache(scientificName);
1108
1109                         if (!config.getDoReUseTaxon() || taxon == null){
1110                                 getNameService().save(taxonName);
1111                                 taxon = Taxon.NewInstance(taxonName, sec); //TODO sec set null
1112                         }
1113                         determinationEvent = DeterminationEvent.NewInstance();
1114                         determinationEvent.setTaxon(taxon);
1115                         determinationEvent.setPreferredFlag(preferredFlag);
1116
1117                         for (String strReference : dataHolder.referenceList){
1118
1119                                 ReferenceBase reference = ReferenceFactory.newGeneric();
1120                                 reference.setTitleCache(strReference, true);
1121                                 determinationEvent.addReference(reference);
1122                         }
1123                         facade.addDetermination(determinationEvent);
1124                 }
1125
1126         }
1127
1128         private NonViralName<?> parseScientificName(String scientificName, Abcd206DataHolder dataHolder){
1129                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1130                 NonViralName<?>taxonName = null;
1131                 boolean problem=false;
1132
1133                 if (dataHolder.nomenclatureCode.toString().equals("Zoological")){
1134                         taxonName = (ZoologicalName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICZN,null);
1135                         if (taxonName.hasProblem()){
1136                                 problem=true;
1137                         }
1138                 }
1139                 if (dataHolder.nomenclatureCode.toString().equals("Botanical")){
1140                         taxonName  = (BotanicalName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICBN,null);
1141                         if (taxonName.hasProblem()){
1142                                 problem=true;;
1143                         }
1144                 }
1145                 if (dataHolder.nomenclatureCode.toString().equals("Bacterial")){
1146                         taxonName = (BacterialName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNB, null);
1147                         if (taxonName.hasProblem()){
1148                                 problem=true;
1149                         }
1150                 }
1151                 if (dataHolder.nomenclatureCode.toString().equals("Cultivar")){
1152                         taxonName = (CultivarPlantName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNCP, null);
1153                         if (taxonName.hasProblem()){
1154                                 problem=true;
1155                         }
1156                 }
1157 //              if (this.nomenclatureCode.toString().equals("Viral")){
1158 //              ViralName taxonName = (ViralName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICVCN(), null);
1159 //              if (taxonName.hasProblem())
1160 //              logger.info("pb ICVCN");
1161 //              }
1162                 //TODO: parsing of ViralNames?
1163                 if(problem){
1164                         taxonName = NonViralName.NewInstance(null);
1165                         taxonName.setTitleCache(scientificName, true);
1166                 }
1167                 return taxonName;
1168
1169         }
1170
1171         private NonViralName<?> setTaxonNameByType(HashMap<String, String> atomisedMap,String fullName, Abcd206DataHolder dataHolder){
1172                 if (dataHolder.nomenclatureCode.equals("Zoological")){
1173                         NonViralName<ZoologicalName> taxonName  = ZoologicalName.NewInstance(null);
1174                         taxonName.setFullTitleCache(fullName, true);
1175                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1176                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"SubGenus"));
1177                         taxonName.setSpecificEpithet(getFromMap(atomisedMap,"SpeciesEpithet"));
1178                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"SubspeciesEpithet"));
1179                         Team team = null;
1180                         if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1181                                 team = Team.NewInstance();
1182                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"), true);
1183                         }else{
1184                                 if (getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1185                                         team = Team.NewInstance();
1186                                         team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamAndYear"), true);
1187                                 }
1188                         }
1189                         if(team != null){
1190                                 taxonName.setBasionymAuthorTeam(team);
1191                         }else{
1192                                 if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1193                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"));
1194                                 } else if (getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1195                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamAndYear"));
1196                                 }
1197                         }
1198                         if(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear") != null){
1199                                 team = Team.NewInstance();
1200                                 team.setTitleCache(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear"), true);
1201                                 taxonName.setCombinationAuthorTeam(team);
1202                         }
1203                         if (taxonName.hasProblem()){
1204                                 logger.info("pb ICZN");
1205                         }else{
1206                                 return taxonName;
1207                         }
1208                 }
1209                 if (dataHolder.nomenclatureCode.equals("Botanical")){
1210                         NonViralName<BotanicalName> taxonName  = BotanicalName.NewInstance(null);
1211                         taxonName.setFullTitleCache(fullName, true);
1212                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1213                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"FirstEpithet"));
1214                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"InfraSpeEpithet"));
1215                         try{taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap,"Rank")));
1216                         }catch(Exception e){}
1217                         Team team = null;
1218                         if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1219                                 team = Team.NewInstance();
1220                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"), true);
1221                                 if(team != null){
1222                                         taxonName.setBasionymAuthorTeam(team);
1223                                 }
1224                         }
1225                         if (getFromMap(atomisedMap,"AuthorTeam") != null){
1226                                 team = Team.NewInstance();
1227                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeam"), true);
1228                                 if(team != null){
1229                                         taxonName.setCombinationAuthorTeam(team);
1230                                 }
1231                         }
1232                         if (team == null)       {
1233                                 if(getFromMap(atomisedMap,"AuthorTeamParenthesis") != null){
1234                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeamParenthesis"));
1235                                 }else if (getFromMap(atomisedMap,"AuthorTeam") != null){
1236                                         taxonName.setAuthorshipCache(getFromMap(atomisedMap,"AuthorTeam"));
1237                                 }
1238                         }
1239                         if(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear") != null){
1240                                 team = Team.NewInstance();
1241                                 team.setTitleCache(getFromMap(atomisedMap,"CombinationAuthorTeamAndYear"), true);
1242                                 taxonName.setCombinationAuthorTeam(team);
1243                         }
1244                         if (taxonName.hasProblem()){
1245                                 logger.info("pb ICBN");
1246                         }else {
1247                                 return taxonName;
1248                         }
1249                 }
1250                 if (dataHolder.nomenclatureCode.equals("Bacterial")){
1251                         NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
1252                         taxonName.setFullTitleCache(fullName, true);
1253                         taxonName.setGenusOrUninomial(getFromMap(atomisedMap,"Genus"));
1254                         taxonName.setInfraGenericEpithet(getFromMap(atomisedMap,"SubGenus"));
1255                         taxonName.setSpecificEpithet(getFromMap(atomisedMap,"Species"));
1256                         taxonName.setInfraSpecificEpithet(getFromMap(atomisedMap,"SubspeciesEpithet"));
1257                         if(getFromMap(atomisedMap,"AuthorTeamAndYear") != null){
1258                                 Team team = Team.NewInstance();
1259                                 team.setTitleCache(getFromMap(atomisedMap,"AuthorTeamAndYear"), true);
1260                                 taxonName.setCombinationAuthorTeam(team);
1261                         }
1262                         if(getFromMap(atomisedMap,"ParentheticalAuthorTeamAndYear") != null){
1263                                 Team team = Team.NewInstance();
1264                                 team.setTitleCache(getFromMap(atomisedMap,"ParentheticalAuthorTeamAndYear"), true);
1265                                 taxonName.setBasionymAuthorTeam(team);
1266                         }
1267                         if (taxonName.hasProblem()){
1268                                 logger.info("pb ICNB");
1269                         }else{
1270                                 return taxonName;
1271                         }
1272                 }
1273                 if (dataHolder.nomenclatureCode.equals("Cultivar")){
1274                         CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
1275
1276                         if (taxonName.hasProblem()){
1277                                 logger.info("pb ICNCP");
1278                         }else {
1279                                 return taxonName;
1280                         }
1281                 }
1282 //              if (this.nomenclatureCode.equals("Viral")){
1283 //              ViralName taxonName = ViralName.NewInstance(null);
1284 //              taxonName.setFullTitleCache(fullName, true);
1285 //              taxonName.setAcronym(getFromMap(atomisedMap,"Acronym"));
1286 //              if (taxonName.hasProblem())
1287 //              logger.info("pb ICVCN");
1288 //              else return taxonName;
1289 //              }
1290                 //TODO ViralName
1291                 NonViralName<?>taxonName = NonViralName.NewInstance(null);
1292                 taxonName.setFullTitleCache(fullName, true);
1293                 return taxonName;
1294         }
1295
1296         private String getFromMap(HashMap<String, String> atomisedMap, String key){
1297                 String value = null;
1298                 if (atomisedMap.containsKey(key)){
1299                         value = atomisedMap.get(key);
1300                 }
1301                 try{
1302                         if (value != null && key.matches(".*Year.*")){
1303                                 value=value.trim();
1304                                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")){
1305                                         String tmp=value.split("[0-9]{4}$")[0];
1306                                         int year = Integer.parseInt(value.split(tmp)[1]);
1307                                         if (year >= 1752){
1308                                                 value=tmp;
1309                                         }else{
1310                                                 value=null;
1311                                         }
1312                                 }else{
1313                                         value=null;
1314                                 }
1315                         }
1316                 }catch(Exception e){value=null;}
1317
1318                 return value;
1319         }
1320
1321         private void compareABCDtoCDM(String urlFileName, ArrayList<String> knownElts, Abcd206DataHolder dataHolder){
1322
1323                 try {
1324                         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
1325                         DocumentBuilder constructeur = factory.newDocumentBuilder();
1326                         URL url = new URL(urlFileName);
1327                         Object o = url.getContent();
1328                         InputStream is = (InputStream)o;
1329                         Document document = constructeur.parse(is);
1330                         Element root = document.getDocumentElement();
1331                         traverse(root, dataHolder);
1332                 } catch (ParserConfigurationException e) {
1333                         e.printStackTrace();
1334                 } catch (SAXException e) {
1335                         e.printStackTrace();
1336                 } catch (IOException e) {
1337                         e.printStackTrace();
1338                 }
1339                 Set<String> elts = dataHolder.allABCDelements.keySet();
1340                 Iterator< String>it = elts.iterator();
1341                 String elt;
1342                 while (it.hasNext()){
1343                         elt = it.next();
1344                         if (knownElts.indexOf(elt) == -1){
1345                                 logger.info("Unsaved ABCD element: " + elt + " - " + dataHolder.allABCDelements.get(elt));
1346                         }
1347                 }
1348         }
1349
1350
1351
1352         /**
1353          * Traverses the tree for compareABCDtoCDM
1354          * @param node
1355          * @param dataHolder
1356          */
1357         private void traverse(Node node, Abcd206DataHolder dataHolder){
1358                 // Extract node info:
1359                 String test = node.getTextContent();
1360
1361                 // Print and continue traversing.
1362                 if(test != null && test != "#text" && node.getNodeName() != "#text" && test.split("\n").length==1 && test.length()>0){
1363                         path=node.getNodeName();
1364                         getHierarchie(node);
1365                         dataHolder.allABCDelements.put(path,test);
1366                         path="";
1367                 }
1368                 // Now traverse the rest of the tree in depth-first order.
1369                 if (node.hasChildNodes()) {
1370                         // Get the children in a list.
1371                         NodeList nl = node.getChildNodes();
1372                         // How many of them?
1373                         int size = nl.getLength();
1374                         for (int i=0; i<size; i++){
1375                                 // Recursively traverse each of the children.
1376                                 traverse (nl.item(i), dataHolder);
1377                         }
1378                 }
1379         }
1380
1381
1382
1383         @Override
1384         protected boolean isIgnore(Abcd206ImportState state) {
1385                 //return ! config.isDoNameFacts();
1386                 return false;
1387         }
1388
1389
1390
1391 }