cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenSythesysExcelImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9 package eu.etaxonomy.cdm.io.specimen.excel.in;
  10
  11 import java.io.FileNotFoundException;
  12 import java.net.MalformedURLException;
  13 import java.util.ArrayList;
  14 import java.util.Calendar;
  15 import java.util.HashMap;
  16 import java.util.HashSet;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.Scanner;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import org.apache.commons.lang3.StringUtils;
  24 import org.apache.logging.log4j.LogManager;
  25 import org.apache.logging.log4j.Logger;
  26 import org.springframework.stereotype.Component;
  27 import org.springframework.transaction.TransactionStatus;
  28
  29 import eu.etaxonomy.cdm.common.ExcelUtils;
  30 import eu.etaxonomy.cdm.common.URI;
  31 import eu.etaxonomy.cdm.facade.DerivedUnitFacade;
  32 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  33 import eu.etaxonomy.cdm.io.common.ICdmIO;
  34 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  35 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  36 import eu.etaxonomy.cdm.model.agent.AgentBase;
  37 import eu.etaxonomy.cdm.model.agent.Institution;
  38 import eu.etaxonomy.cdm.model.agent.Person;
  39 import eu.etaxonomy.cdm.model.agent.Team;
  40 import eu.etaxonomy.cdm.model.common.CdmBase;
  41 import eu.etaxonomy.cdm.model.common.Language;
  42 import eu.etaxonomy.cdm.model.common.TimePeriod;
  43 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  44 import eu.etaxonomy.cdm.model.description.Feature;
  45 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  46 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  47 import eu.etaxonomy.cdm.model.location.NamedArea;
  48 import eu.etaxonomy.cdm.model.media.Media;
  49 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  50 import eu.etaxonomy.cdm.model.name.TaxonName;
  51 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  52 import eu.etaxonomy.cdm.model.occurrence.Collection;
  53 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  54 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  55 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  56 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  57 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  58 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  59 import eu.etaxonomy.cdm.model.reference.Reference;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  61 import eu.etaxonomy.cdm.model.taxon.Classification;
  62 import eu.etaxonomy.cdm.model.taxon.Taxon;
  63 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  64 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  65 import eu.etaxonomy.cdm.model.term.DefinedTermBase;
  66 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  67 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  68 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  69
  70 /**
  71  * @author p.kelbert
  72  * @since 29.10.2008
  73  * @since 13 mars 2013
  74  */
  75 @Component
  76 public class SpecimenSythesysExcelImport  extends CdmImportBase<SpecimenSynthesysExcelImportConfigurator, SpecimenSynthesysExcelImportState>
  77         implements ICdmIO<SpecimenSynthesysExcelImportState> {
  78
  79     private static final long serialVersionUID = -1145031415387024364L;
  80     private static final Logger logger = LogManager.getLogger();
  81
  82     protected String fullScientificNameString;
  83     protected String nomenclatureCode;
  84     protected String institutionCode;
  85     protected String collectionCode;
  86     protected String unitID;
  87     protected String recordBasis;
  88     protected String accessionNumber;
  89     protected String fieldNumber;
  90     protected Double longitude;
  91     protected Double latitude;
  92     protected String locality;
  93     protected String languageIso = null;
  94     protected String country;
  95     protected String isocountry;
  96     protected int depth;
  97     protected int altitude;
  98     protected String gatheringYear;
  99     protected String gatheringMonth;
 100     protected String gatheringDay;
 101     protected String gatheringDate;
 102     protected String gatheringTeam;
 103     protected String gatheringAgent;
 104     protected String originalsource;
 105     protected String identifier;
 106     protected String gatheringNotes;
 107
 108     private DerivedUnit derivedUnitBase;
 109     private Reference ref = null;
 110     private TransactionStatus tx;
 111     private Classification classification = null;
 112
 113     protected ArrayList<String> identificationList = new ArrayList<>();
 114     protected ArrayList<String> multimediaObjects = new ArrayList<>();
 115     private boolean keepAtomisedDate=true;
 116     private boolean useTDWGarea = true;
 117
 118     boolean DEBUG =false;
 119
 120      public SpecimenSythesysExcelImport() {
 121         super();
 122     }
 123
 124     /**
 125      * private HashMap that handle null values (missing keys)
 126      * return empty string instead of null
 127      * */
 128     public class MyHashMap<K,V> extends HashMap<K,V> {
 129
 130         private static final long serialVersionUID = -6230407405666753405L;
 131
 132         @SuppressWarnings("unchecked")
 133         @Override
 134         public V get(Object key) {
 135             Object a;
 136             if (containsKey(key)) {
 137                 a= super.get(key);
 138             } else {
 139                 a="";
 140             }
 141             if (a ==null || a.toString().equalsIgnoreCase("none")) {
 142                 a="";
 143             }
 144             return (V) a;
 145         }
 146     }
 147
 148
 149     /**
 150      * getClassification : get the classification declared in the ImportState
 151      *
 152      * @param state
 153      * @return
 154      */
 155     private void setClassification(SpecimenSynthesysExcelImportState state) {
 156         if (classification == null){
 157             boolean configreused=false;
 158             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Goosefoots")) {
 159                 classification = getClassificationService().find(UUID.fromString("2c2dc41c-9891-42cd-9cd5-8b28dfdd1b8a"));
 160                 if (classification ==null){
 161                     String name = state.getConfig().getClassificationName();
 162                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 163                     getClassificationService().saveOrUpdate(classification);
 164                 }
 165                 configreused=true;
 166             }
 167             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Ants")) {
 168                 classification = getClassificationService().find(UUID.fromString("9b72d32f-2868-4f0d-81bd-ebc38ee8c645"));
 169                 if (classification ==null){
 170                     String name = state.getConfig().getClassificationName();
 171                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 172                     getClassificationService().saveOrUpdate(classification);
 173                 }
 174                 configreused=true;
 175             }
 176             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Fungi")) {
 177                 classification = getClassificationService().find(UUID.fromString("fe07274d-daff-4678-9f7d-2c3916948c3e"));
 178                 if (classification ==null){
 179                     String name = state.getConfig().getClassificationName();
 180                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 181                     getClassificationService().saveOrUpdate(classification);
 182                 }
 183                 configreused=true;
 184             }
 185             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Campylopus")) {
 186                 classification = getClassificationService().find(UUID.fromString("9dccbd11-eb0e-4cac-ae04-aa61ad82b8a6"));
 187                 if (classification ==null){
 188                     String name = state.getConfig().getClassificationName();
 189                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 190                     getClassificationService().saveOrUpdate(classification);
 191                 }
 192                 configreused=true;
 193             }
 194
 195             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Eupolybothrus")) {
 196                 classification = getClassificationService().find(UUID.fromString("62ce65e0-e036-49aa-aae7-4d4d0f88966c"));
 197                 if (classification ==null){
 198                     String name = state.getConfig().getClassificationName();
 199                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 200                     getClassificationService().saveOrUpdate(classification);
 201                 }
 202                 configreused=true;
 203             }
 204             if(!configreused){
 205
 206                 String name = state.getConfig().getClassificationName();
 207                 classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 208                 if (state.getConfig().getClassificationUuid() != null) {
 209                     classification.setUuid(state.getConfig().getClassificationUuid());
 210                 }
 211                 getClassificationService().saveOrUpdate(classification);
 212             }
 213         }
 214     }
 215
 216     /**
 217      * refresh the hibernate transaction :
 218      * - commit the current queries
 219      * - get the reference and the classification and the derivedUnitBase back from the hibernate session
 220      * */
 221     private void refreshTransaction(){
 222         commitTransaction(tx);
 223         tx = startTransaction();
 224         ref = getReferenceService().find(ref.getUuid());
 225         classification = getClassificationService().find(classification.getUuid());
 226         try{
 227             derivedUnitBase = (DerivedUnit) getOccurrenceService().find(derivedUnitBase.getUuid());
 228         }catch(Exception e){
 229             //logger.warn("derivedunit up to date or not created yet");
 230         }
 231     }
 232
 233     //    /**
 234     //     * refresh the hibernate transaction :
 235     //     * - commit the current queries
 236     //     * - get the reference and the classification and the derivedUnitBase back from the hibernate session
 237     //     * */
 238     //    private void refreshTransac(){
 239     //        commitTransaction(tx);
 240     //        tx = startTransaction();
 241     //    }
 242     /*
 243      * Store the unit's properties into variables
 244      * @param unit: the hashmap containing the splitted Excel line (Key=column name, value=value)
 245      */
 246     private void setUnitPropertiesExcel(MyHashMap<String,String> unit, String defaultAuthor){
 247         multimediaObjects = new ArrayList<String>();
 248         identificationList = new ArrayList<String>();
 249         gatheringTeam="";
 250         gatheringAgent="";
 251
 252         String author = unit.get("author");
 253         if (author.isEmpty() && !defaultAuthor.isEmpty()) {
 254             author=defaultAuthor;
 255         }
 256         String taxonName = unit.get("taxonName");
 257
 258         institutionCode = unit.get("institution");
 259         collectionCode = unit.get("collection");
 260         unitID = unit.get("unitID");
 261         recordBasis = unit.get("recordBasis");
 262
 263         accessionNumber = null;
 264
 265         try {longitude = Double.valueOf(unit.get("longitude"));
 266         } catch (Exception e) {longitude = 0.0;}
 267
 268         try {latitude = Double.valueOf(unit.get("latitude"));
 269         } catch (Exception e) {latitude = 0.0;}
 270
 271         country = unit.get("country");
 272         isocountry = unit.get("isoCountry");
 273         locality = unit.get("locality");
 274         String habitat = unit.get("habitat");
 275         String stateProvince = unit.get("stateProvince");
 276         if (!stateProvince.isEmpty()) {
 277             locality=stateProvince+", "+locality;
 278         }
 279         if (!habitat.isEmpty()){
 280             locality+=" (Habitat: "+habitat+")";
 281         }
 282
 283         fieldNumber = unit.get("field number");
 284
 285         String url =unit.get("url");
 286         if (!url.isEmpty()) {
 287             multimediaObjects.add(url);
 288         }
 289
 290         String coll =unit.get("collector");
 291         if (!coll.isEmpty()) {
 292             if (coll.indexOf("& al.")!=-1 || coll.indexOf("et al.") != -1 || coll.indexOf(" al. ")!=-1 ||
 293                     coll.indexOf("&") != -1 || coll.indexOf(" et ") != -1 || coll.indexOf(";")!=-1) {
 294                 gatheringTeam = coll;
 295             } else{
 296                 //single
 297                 gatheringAgent = coll;
 298             }
 299         }
 300
 301         if(!taxonName.contains(author)) {
 302             identificationList.add(taxonName+" "+author);
 303         } else {
 304             identificationList.add(taxonName);
 305         }
 306
 307         gatheringYear = unit.get("year");
 308         gatheringMonth = unit.get("month");
 309         gatheringDay = unit.get("day");
 310         gatheringDate = unit.get("date");
 311
 312         gatheringNotes = unit.get("eventRemarks");
 313         originalsource = unit.get("originalsource");
 314         identifier = unit.get("identifier");
 315     }
 316
 317     private Institution getInstitution(SpecimenSynthesysExcelImportConfigurator config){
 318         Institution institution;
 319         List<Institution> institutions;
 320         try{
 321             institutions= getAgentService().searchInstitutionByCode(institutionCode);
 322         }catch(Exception e){
 323             institutions=new ArrayList<Institution>();
 324         }
 325         if (institutions.size() ==0 || !config.getReUseExistingMetadata()){
 326             logger.debug("Institution (agent) unknown or not allowed to reuse existing metadata");
 327             //create institution
 328             institution = Institution.NewInstance();
 329             institution.setCode(institutionCode);
 330             getAgentService().saveOrUpdate(institution);
 331         }
 332         else{
 333             logger.debug("Institution (agent) already in the db");
 334             institution = institutions.get(0);
 335         }
 336         return CdmBase.deproxy(institution, Institution.class);
 337     }
 338
 339     /*
 340      * Look if the Collection does already exists
 341      * @param collectionCode: a string
 342      * @param institution: the current Institution
 343      * @param app
 344      * @return the Collection (existing or new)
 345      */
 346     private Collection getCollection(Institution institution, SpecimenSynthesysExcelImportConfigurator config){
 347         Collection collection =null;
 348         List<Collection> collections;
 349         try{
 350             collections = getCollectionService().searchByCode(collectionCode);
 351         }catch(Exception e){
 352             collections=new ArrayList<Collection>();
 353         }
 354         if (collections.size() ==0 || !config.getReUseExistingMetadata()){
 355             logger.debug("Collection not found or do not reuse existing metadata  "+collectionCode);
 356             //create new collection
 357             collection= Collection.NewInstance();
 358             collection.setCode(collectionCode);
 359             collection.setCodeStandard("GBIF");
 360             collection.setInstitute(institution);
 361         }
 362         else{
 363             boolean collectionFound=false;
 364             for (int i=0; i<collections.size(); i++){
 365                 collection = collections.get(i);
 366                 try {
 367                     if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())){
 368                         //found a collection with the same code and the same institution
 369                         collectionFound=true;
 370                     }
 371                 } catch (NullPointerException e) {}
 372             }
 373             if (!collectionFound){
 374                 collection= Collection.NewInstance();
 375                 collection.setCode(collectionCode);
 376                 collection.setCodeStandard("GBIF");
 377                 collection.setInstitute(institution);
 378             }
 379
 380         }
 381         return collection;
 382     }
 383
 384     private void setTaxonName(SpecimenSynthesysExcelImportConfigurator config){
 385         TaxonName taxonName = null;
 386         Taxon taxon = null;
 387
 388         String scientificName="";
 389         boolean preferredFlag=false;
 390
 391         for (int i = 0; i < identificationList.size(); i++) {
 392             String fullScientificNameString = identificationList.get(i);
 393             fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
 394             if (fullScientificNameString.indexOf("_preferred_") != -1){
 395                 scientificName = fullScientificNameString.split("_preferred_")[0].trim();
 396                 String pTmp = fullScientificNameString.split("_preferred_")[1].split("_code_")[0];
 397                 if (pTmp == "1" || pTmp.toLowerCase().indexOf("true") != -1) {
 398                     preferredFlag=true;
 399                 } else {
 400                     preferredFlag=false;
 401                 }
 402             }else {
 403                 scientificName = fullScientificNameString.trim();
 404             }
 405
 406             if (fullScientificNameString.indexOf("_code_") != -1){
 407                 nomenclatureCode = fullScientificNameString.split("_code_")[1].trim();
 408             }
 409
 410             if (config.getDoReUseTaxon()){
 411                 List<TaxonBase> c = null;
 412                 try {
 413                     Taxon cc= getTaxonService().findBestMatchingTaxon(scientificName);
 414                     if (cc != null){
 415                         if ((cc.getSec() == null || cc.getSec().toString().isEmpty()) || (cc.getSec() != null &&
 416                                 cc.getSec().getTitleCache().equalsIgnoreCase(ref.getTitleCache()))) {
 417                             if(cc.getSec() == null || cc.getSec().toString().isEmpty()){
 418                                 cc.setSec(ref);
 419                                 getTaxonService().saveOrUpdate(cc);
 420                             }
 421                             taxon=cc;
 422                         }
 423                     }
 424                     else{
 425                         c = getTaxonService().searchByName(scientificName, INCLUDE_UNPUBLISHED, ref);
 426                         for (TaxonBase<?> b : c) {
 427                             taxon = (Taxon) b;
 428                         }
 429                     }
 430                 } catch (Exception e) {
 431                     logger.info("Searchtaxabyname failed" + e);
 432                     taxon = null;
 433                 }
 434
 435             }
 436             if (!config.getDoReUseTaxon() || taxon == null){
 437                 if (DEBUG) {
 438                     logger.info("create new taxonName instance");
 439                 }
 440                 if (config.getDoAutomaticParsing()){
 441                     taxonName = parseScientificName(scientificName);
 442                 }
 443                 else{
 444                     taxonName = TaxonNameFactory.NewNonViralInstance(null);
 445                     taxonName.setTitleCache(scientificName, true);
 446                 }
 447                 getNameService().save(taxonName);
 448                 taxon = Taxon.NewInstance(taxonName, ref); //sec set null
 449                 getTaxonService().save(taxon);
 450                 //   refreshTransaction();
 451                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 452                 taxon = addTaxonNode(taxon, config);
 453             }
 454
 455             DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 456             determinationEvent.setTaxon(getTaxonService().find(taxon.getUuid()));
 457             determinationEvent.setPreferredFlag(preferredFlag);
 458
 459             determinationEvent.setIdentifiedUnit(derivedUnitBase);
 460             if (!identifier.isEmpty()){
 461                 Person p = Person.NewInstance();
 462                 p.setTitleCache(identifier, true);
 463                 determinationEvent.setActor(p);
 464             }
 465             derivedUnitBase.addDetermination(determinationEvent);
 466
 467             makeIndividualsAssociation(taxon,determinationEvent,config);
 468
 469             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 470         }
 471     }
 472
 473     private Taxon addTaxonNode(Taxon taxon, SpecimenSynthesysExcelImportConfigurator config) {
 474         if (DEBUG) {
 475             logger.info("link taxon to a taxonNode");
 476         }
 477         boolean exist = false;
 478         for (TaxonNode p : classification.getAllNodes()){
 479             if(p.getTaxon().equals(taxon)) {
 480                 exist =true;
 481             }
 482         }
 483         if (!exist){
 484             taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 485             classification.addChildTaxon(taxon, ref, null);
 486             getClassificationService().saveOrUpdate(classification);
 487             //            refreshTransaction();
 488         }
 489         return (Taxon) getTaxonService().find(taxon.getUuid());
 490     }
 491
 492     private TaxonName parseScientificName(String scientificName){
 493         if (DEBUG) {
 494             logger.debug("in parseScientificName");
 495         }
 496         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 497         TaxonName taxonName = null;
 498         boolean problem=false;
 499
 500         if (DEBUG) {
 501             logger.debug("nomenclature: "+nomenclatureCode);
 502         }
 503
 504         if(nomenclatureCode == null){
 505             taxonName = TaxonNameFactory.NewNonViralInstance(null);
 506             taxonName.setTitleCache(scientificName, true);
 507             return taxonName;
 508         }
 509
 510         if (nomenclatureCode.toString().equals("Zoological")){
 511             taxonName = (TaxonName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICZN,null);
 512             if (taxonName.hasProblem()) {
 513                 problem=true;
 514             }
 515         }
 516         if (nomenclatureCode.toString().equals("Botanical")){
 517             taxonName  = (TaxonName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNAFP,null);
 518             if (taxonName.hasProblem()) {
 519                 problem=true;
 520             }}
 521         if (nomenclatureCode.toString().equals("Bacterial")){
 522             taxonName = (TaxonName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNP, null);
 523             if (taxonName.hasProblem()) {
 524                 problem=true;
 525             }
 526         }
 527         if (nomenclatureCode.toString().equals("Cultivar")){
 528             taxonName = (TaxonName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNCP, null);
 529             if (taxonName.hasProblem()) {
 530                 problem=true;
 531             }
 532         }
 533         //      if (nomenclatureCode.toString().equals("Viral")){
 534         //      ViralName taxonName = (ViralName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICVCN(), null);
 535         //      if (taxonName.hasProblem())
 536         //      System.out.println("pb ICVCN");
 537         //      }
 538         //TODO: parsing of ViralNames?
 539         if(problem){
 540             taxonName = TaxonNameFactory.NewNonViralInstance(null);
 541             taxonName.setTitleCache(scientificName, true);
 542         }
 543         return taxonName;
 544
 545     }
 546
 547     private DerivedUnitFacade getFacade() {
 548         // logger.info("GETFACADE");
 549         /**
 550          * SPECIMEN OR OBSERVATION OR LIVING
 551          */
 552         // DerivedUnit derivedThing = null;
 553         SpecimenOrObservationType type = null;
 554
 555         // create specimen
 556         if (recordBasis != null) {
 557             String rec = recordBasis.toLowerCase();
 558             if (rec.contains("specimen") || rec.startsWith("s")) {// specimen
 559                 type = SpecimenOrObservationType.PreservedSpecimen;
 560             }
 561             if (rec.contains("observat") || rec.startsWith("o")) {
 562                 type = SpecimenOrObservationType.Observation;
 563             }
 564             if (rec.contains("fossil") || rec.startsWith("f") ){
 565                 type = SpecimenOrObservationType.Fossil;
 566             }
 567
 568             if (rec.contains("living") || rec.startsWith("l")) {
 569                 type = SpecimenOrObservationType.LivingSpecimen;
 570             }
 571             if (type == null) {
 572                 if(DEBUG) {
 573                     logger.info("The basis of record does not seem to be known: "   + recordBasis);
 574                 }
 575                 type = SpecimenOrObservationType.DerivedUnit;
 576             }
 577         } else {
 578             logger.info("The basis of record is null");
 579             type = SpecimenOrObservationType.DerivedUnit;
 580         }
 581         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 582         return derivedUnitFacade;
 583     }
 584
 585     /*
 586      * Store the unit with it's Gathering informations in the CDM
 587      */
 588     public boolean start(SpecimenSynthesysExcelImportConfigurator config){
 589         boolean result = true;
 590
 591         //        refreshTransaction();
 592         try {
 593
 594             /**
 595              * SPECIMEN OR OBSERVATION OR LIVING
 596              */
 597             DerivedUnitFacade derivedUnitFacade = getFacade();
 598             derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 599
 600             //set catalogue number (unitID)
 601             derivedUnitFacade.setCatalogNumber(unitID);
 602             derivedUnitFacade.setAccessionNumber(accessionNumber);
 603
 604             if (!originalsource.isEmpty()){
 605                 Reference reference = ReferenceFactory.newGeneric();
 606                 reference.setTitleCache(originalsource, true);
 607                 derivedUnitBase.addSource(OriginalSourceType.Import, originalsource, "", reference, "");
 608             }else{
 609                 derivedUnitBase.addSource(OriginalSourceType.Import, unitID,"",ref,ref.getCitation());
 610             }
 611             /**
 612              * INSTITUTION & COLLECTION
 613              */
 614             //manage institution
 615             if (!institutionCode.isEmpty()){
 616                 Institution institution = getInstitution(config);
 617                 //manage collection
 618                 if (!collectionCode.isEmpty()){
 619                     Collection collection = getCollection(institution, config);
 620                     //link specimen & collection
 621                     derivedUnitFacade.setCollection(collection);
 622                 }
 623             }
 624
 625             /**
 626              * GATHERING EVENT
 627              */
 628             // gathering event
 629             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(getTermService(), locality, languageIso,
 630                     longitude, latitude, gatheringAgent, gatheringTeam,config);
 631             // country
 632             UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea();
 633             unitsGatheringArea.useTDWGareas(this.useTDWGarea);
 634             //            unitsGatheringArea.setConfig(config, getOccurrenceService(),getTermService());
 635             unitsGatheringArea.setParams(isocountry, country, config, getTermService(), getVocabularyService());
 636             DefinedTermBase areaCountry =unitsGatheringArea.getCountry();
 637
 638
 639
 640             // copy gathering event to facade
 641             GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 642             //join gatheringEvent to fieldObservation
 643             derivedUnitFacade.setGatheringEvent(gatheringEvent);
 644
 645             derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 646             derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 647             //derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 648             derivedUnitFacade.setCountry((NamedArea)areaCountry);
 649             for(DefinedTermBase<?> area:unitsGatheringArea.getAreas()){
 650                 derivedUnitFacade.addCollectingArea((NamedArea) area);
 651             }
 652             if (gatheringNotes != null && !gatheringNotes.isEmpty()) {
 653                 derivedUnitFacade.setFieldNotes(gatheringNotes);
 654             }
 655
 656
 657             /*
 658              * merge AND STORE DATA
 659              */
 660             //            getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 661             getTermService().saveLanguageData(unitsGatheringEvent.getLocality());
 662
 663
 664
 665             //add gathering date
 666             if (keepAtomisedDate && (!gatheringYear.isEmpty() || !gatheringMonth.isEmpty() || !gatheringDay.isEmpty())){
 667                 Calendar calendar =  Calendar.getInstance();
 668                 if (!gatheringYear.isEmpty()) {
 669                     TimePeriod tp = TimePeriod.NewInstance(Integer.parseInt(gatheringYear));
 670                     if (!gatheringMonth.isEmpty()) {
 671                         tp.setStartMonth(Integer.parseInt(gatheringMonth));
 672                         if (!gatheringDay.isEmpty()) {
 673                             tp.setStartDay(Integer.parseInt(gatheringDay));
 674                         }
 675                     }
 676                     unitsGatheringEvent.setGatheringDate(tp);
 677                 }
 678
 679             }else{
 680                 if (!gatheringDate.isEmpty()){
 681                     TimePeriod tp = TimePeriodParser.parseString(gatheringDate);
 682                     unitsGatheringEvent.setGatheringDate(tp);
 683                 }
 684             }
 685
 686
 687             //add fieldNumber
 688             derivedUnitFacade.setFieldNumber(fieldNumber);
 689
 690             //add Multimedia URLs
 691             if(multimediaObjects.size()>0){
 692                 for (String multimediaObject : multimediaObjects) {
 693                     Media media;
 694                     try {
 695                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA);
 696                         derivedUnitFacade.addDerivedUnitMedia(media);
 697                     } catch (MalformedURLException e) {
 698                         // TODO Auto-generated catch block
 699                         e.printStackTrace();
 700                     }
 701                 }
 702             }
 703
 704             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 705
 706             setTaxonName(config);
 707
 708             //         refreshTransaction();
 709             if (DEBUG) {
 710                 logger.info("saved new specimen ...");
 711             }
 712
 713         } catch (Exception e) {
 714             logger.warn("Error when reading record!!");
 715             e.printStackTrace();
 716             result = false;
 717         }
 718         if (DEBUG) {
 719             logger.info("commit done");
 720         }
 721         //app.close();
 722         return result;
 723     }
 724
 725     /*
 726      * Store the unit with it's Gathering informations in the CDM
 727      */
 728     public boolean resetCollectionInstitution(SpecimenSynthesysExcelImportConfigurator config,List<SpecimenOrObservationBase> specimenOrObs){
 729         boolean result = true;
 730
 731
 732         for (SpecimenOrObservationBase<?> specimen:specimenOrObs){
 733             DerivedUnit derivedUnit=null;
 734             if (specimen.isInstanceOf(GatheringEvent.class)){
 735                 System.out.println("gathering");
 736                 GatheringEvent gath = CdmBase.deproxy(specimen, GatheringEvent.class);
 737             }
 738             if (specimen.isInstanceOf(DerivedUnit.class)){
 739                 derivedUnit = CdmBase.deproxy(specimen, DerivedUnit.class);
 740                 String unitIDDB = derivedUnit.getCatalogNumber();
 741                 if (unitIDDB !=null) {
 742                     if(unitIDDB.equalsIgnoreCase(this.unitID)){
 743                         System.out.println("unitID found");
 744
 745                         Institution institution = getInstitution(config);
 746
 747                         Collection col = getCollection(institution, config);
 748                         getCollectionService().saveOrUpdate(col);
 749                         derivedUnit.setCollection(col);
 750                         getOccurrenceService().saveOrUpdate(derivedUnit);
 751                         return result;
 752                     }
 753                 }
 754             }
 755             if(specimen.isInstanceOf(IndividualsAssociation.class)) {
 756                 System.out.println("Indivi assoc");
 757             }
 758         }
 759         return result;
 760     }
 761
 762
 763     private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
 764         if (unit == null){
 765             return null;
 766         }
 767         SpecimenOrObservationType type = unit.getRecordBasis();
 768
 769         if (type.isFeatureObservation()){
 770             return Feature.OBSERVATION();
 771         }else if (type.isPreservedSpecimen() ||
 772                 type == SpecimenOrObservationType.LivingSpecimen ||
 773                 type == SpecimenOrObservationType.OtherSpecimen
 774                 ){
 775             return Feature.SPECIMEN();
 776         }else if (type == SpecimenOrObservationType.Unknown ||
 777                 type == SpecimenOrObservationType.DerivedUnit
 778                 ) {
 779             return Feature.INDIVIDUALS_ASSOCIATION();
 780         }
 781         if (DEBUG) {
 782             logger.warn("No feature defined for derived unit class: "
 783                     + unit.getClass().getSimpleName());
 784         }
 785         return null;
 786     }
 787
 788     private void makeIndividualsAssociation(Taxon taxon, DeterminationEvent determinationEvent, SpecimenSynthesysExcelImportConfigurator config) {
 789         //        try{
 790         //            taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 791         //        }
 792         //        catch(Exception e){
 793         //            //logger.info("taxon uptodate");
 794         //        }
 795
 796         TaxonDescription taxonDescription=null;
 797         if (taxon.getTitleCache().contains("Chenopodium vulvaria L.")) {
 798             taxonDescription = (TaxonDescription) getDescriptionService().find(UUID.fromString("a60074e7-979b-41fd-ad4d-d391db474ac4"));
 799         }
 800         if(!taxon.getTitleCache().contains("Chenopodium vulvaria L.") || taxonDescription ==null) {
 801             taxonDescription = getTaxonDescription(taxon, ref, false, true);
 802             taxonDescription.setTitleCache(ref.getTitleCache(), true);
 803         }
 804
 805
 806         taxon.addDescription(taxonDescription);
 807
 808         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
 809         Feature feature = makeFeature(derivedUnitBase);
 810         indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
 811         indAssociation.setFeature(feature);
 812
 813         for (Reference citation : determinationEvent.getReferences()) {
 814             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
 815         }
 816         indAssociation.addSource(OriginalSourceType.Import, null,null,config.getDataReference(),null);
 817
 818         taxonDescription.addElement(indAssociation);
 819
 820         getDescriptionService().saveOrUpdate(taxonDescription);
 821         getTaxonService().saveOrUpdate(taxon);
 822     }
 823
 824     @Override
 825     protected boolean isIgnore(SpecimenSynthesysExcelImportState state) {
 826         return false;
 827     }
 828
 829     @Override
 830     protected void doInvoke(SpecimenSynthesysExcelImportState state) {
 831         boolean success = true;
 832         if (state.getConfig().doAskForDate()) {
 833             keepAtomisedDate = askQuestion("Gathering dates can be stored in either atomised fieds (day month year) or in a concatenated field."+
 834                     "\nWhich value do you want to store?\nPress 1 for the atomised, press 2 for the concatenated field, and then press enter.");
 835             useTDWGarea = askQuestion("Use TDWG area or Country/namedarea?\n Press 1 for TDWG areas\n Press 2 for classics Country.");
 836         }
 837
 838         tx = startTransaction();
 839
 840         ref = state.getConfig().getTaxonReference();
 841         if (ref == null){
 842             ref = state.getConfig().getSourceReference();
 843         }
 844         AgentBase<?> agent= ref.getAuthorship();
 845         if (agent != null){
 846             if (agent.getClass().equals(Team.class)){
 847                 for (Person p : ((Team) agent).getTeamMembers()){
 848                     getAgentService().saveOrUpdate(p);
 849                 }
 850             }
 851             if (agent.getClass().equals(Person.class)){
 852                 getAgentService().saveOrUpdate(agent);
 853             }
 854         }
 855         getReferenceService().saveOrUpdate(ref);
 856         setClassification(state);
 857
 858         refreshTransaction();
 859
 860         URI source = state.getConfig().getSource();
 861         List<Map<String,String>> unitsList = null;
 862         try{
 863             unitsList = ExcelUtils.parseXLS(source);
 864             logger.info("unitslist : "+unitsList.size());
 865         } catch(FileNotFoundException e){
 866             String message = "File not found: " + source;
 867             warnProgress(state, message, e);
 868             logger.error(message);
 869         }
 870
 871         if (unitsList != null){
 872             //load collectors in the database
 873             if (!state.getConfig().isDebugInstitutionOnly()) {
 874                 prepareCollectors(unitsList,state);
 875             }
 876
 877             List<SpecimenOrObservationBase> specimenOrObs = null;
 878             if (state.getConfig().isDebugInstitutionOnly()){
 879                 //DEBUG CHENOPODIUM VULVARIA
 880                 UUID uuid = UUID.fromString("85234ff5-8e40-4813-8f06-44ab960a905a");
 881                 Taxon taxon = (Taxon)getTaxonService().find(uuid);
 882
 883                 specimenOrObs = getOccurrenceService().listByAssociatedTaxon(null, null, taxon, null, null, null, null, null);
 884             }
 885             Map<String,String> unit=null;
 886             MyHashMap<String,String> myunit;
 887             for (int i=0; i<unitsList.size();i++){
 888                 //            for (int i=0; i<10;i++){
 889                 if (i%20==0) {
 890                     logger.info("NbUnit prepared: "+i);
 891                     refreshTransaction();
 892                 }
 893                 unit = unitsList.get(i);
 894                 myunit=new MyHashMap<String, String>();
 895                 for (String key :unit.keySet()) {
 896                     myunit.put(key, unit.get(key));
 897                 }
 898                 System.out.println(myunit.toString());
 899                 //FIXME do this via state
 900                 setUnitPropertiesExcel(myunit, state.getConfig().getDefaultAuthor());//and then invoke
 901                 if (state.getConfig().isDebugInstitutionOnly()){
 902                     success &= resetCollectionInstitution(state.getConfig(),specimenOrObs);
 903                 } else {
 904                     success &= start(state.getConfig());
 905                 }
 906             }
 907         }
 908         if (success == false){
 909             state.setUnsuccessfull();
 910         }
 911         commitTransaction(tx);
 912         return;
 913     }
 914
 915     private void prepareCollectors(List<Map<String, String>> unitsList, SpecimenSynthesysExcelImportState state) {
 916         System.out.println("PREPARE COLLECTORS");
 917         List<String> collectors = new ArrayList<>();
 918         List<String> teams = new ArrayList<>();
 919         List<List<String>> collectorinteams = new ArrayList<>();
 920         String tmp;
 921         for (Map<String,String> unit : unitsList){
 922             tmp=null;
 923             tmp = unit.get("collector");
 924             if (tmp != null && !tmp.isEmpty()) {
 925                 if (tmp.indexOf("et al.") != -1 || tmp.indexOf(" al. ")!=-1 || tmp.indexOf("& al.")!=-1  ) {
 926                     if (!tmp.trim().isEmpty()) {
 927                         teams.add(tmp.trim());
 928                     }
 929                 } else{
 930                     if(tmp.indexOf(" et ")!=-1 || tmp.indexOf("&")!=-1 || tmp.indexOf(";") !=-1){
 931                         List<String> collteam = new ArrayList<String>();
 932                         String[] tmp1 = tmp.split(" et ");
 933                         for (String elt:tmp1){
 934                             String tmp2[] = elt.split("&");
 935                             for (String elt2:tmp2) {
 936                                 if (!elt2.trim().isEmpty()) {
 937                                     String tmp3[] = elt.split(";");
 938                                     for (String elt3:tmp3) {
 939                                         if (!elt3.isEmpty()){
 940                                             collectors.add(elt3.trim());
 941                                             collteam.add(elt3.trim());
 942                                         }
 943                                     }
 944                                 }
 945                             }
 946                         }
 947                         if (collteam.size()>0) {
 948                             collectorinteams.add(new ArrayList<String>(new HashSet<>(collteam)));
 949                         }
 950                     }
 951                     else
 952                         if (!tmp.trim().isEmpty() && !tmp.toString().trim().equalsIgnoreCase("none")) {
 953                             collectors.add(tmp.trim());
 954                         }
 955                 }
 956             }
 957         }
 958
 959         List<String> collectorsU = new ArrayList<String>(new HashSet<>(collectors));
 960         List<String> teamsU = new ArrayList<String>(new HashSet<>(teams));
 961
 962
 963         //existing teams in DB
 964         Map<String,Team> titleCacheTeam = new HashMap<>();
 965         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getUuidAndTitleCache(Team.class, null, null);
 966
 967         Set<UUID> uuids = new HashSet<>();
 968         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
 969             uuids.add(hibernateT.getUuid());
 970         }
 971         if (!uuids.isEmpty()){
 972             List<AgentBase> existingTeams = getAgentService().find(uuids);
 973             for (AgentBase<?> existingP:existingTeams){
 974                 titleCacheTeam.put(existingP.getTitleCache(),(Team) existingP);
 975             }
 976         }
 977
 978         Map<String,UUID> teamMap = new HashMap<>();
 979         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
 980             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
 981         }
 982
 983         //existing persons in DB
 984         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getUuidAndTitleCache(Person.class, null, null);
 985         Map<String,Person> titleCachePerson = new HashMap<>();
 986         uuids = new HashSet<UUID>();
 987         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
 988             uuids.add(hibernateP.getUuid());
 989         }
 990
 991         if (!uuids.isEmpty()){
 992             List<AgentBase> existingPersons = getAgentService().find(uuids);
 993             for (AgentBase<?> existingP:existingPersons){
 994                 titleCachePerson.put(existingP.getTitleCache(),CdmBase.deproxy(existingP, Person.class));
 995             }
 996         }
 997
 998         Map<String,UUID> personMap = new HashMap<String, UUID>();
 999         for (UuidAndTitleCache<Person> person:hiberPersons){
1000             personMap.put(person.getTitleCache(), person.getUuid());
1001         }
1002
1003         java.util.Collection<AgentBase> personsToAdd = new ArrayList<>();
1004         java.util.Collection<AgentBase> teamsToAdd = new ArrayList<>();
1005
1006         for (String collector:collectorsU){
1007             Person p = Person.NewInstance();
1008             p.setTitleCache(collector,true);
1009             if (!personMap.containsKey(p.getTitleCache())){
1010                 personsToAdd.add(p);
1011             }
1012         }
1013         for (String team:teamsU){
1014             Team p = Team.NewInstance();
1015             p.setTitleCache(team,true);
1016             if (!teamMap.containsKey(p.getTitleCache())){
1017                 teamsToAdd.add(p);
1018             }
1019         }
1020         Map<UUID, AgentBase> uuuidPerson = getAgentService().save(personsToAdd);
1021         for (UUID u:uuuidPerson.keySet()){
1022             titleCachePerson.put(uuuidPerson.get(u).getTitleCache(),CdmBase.deproxy(uuuidPerson.get(u), Person.class) );
1023         }
1024
1025         Person ptmp ;
1026         Map <String,Integer>teamdone = new HashMap<String, Integer>();
1027         for (List<String> collteam: collectorinteams){
1028             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
1029                 Team team = new Team();
1030                 boolean em =true;
1031                 for (String collector:collteam){
1032                     ptmp = Person.NewInstance();
1033                     ptmp.setTitleCache(collector,true);
1034                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
1035                     team.addTeamMember(p2);
1036                     em=false;
1037                 }
1038                 if (!em) {
1039                     teamsToAdd.add(team);
1040                 }
1041                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
1042             }
1043         }
1044
1045         Map<UUID, AgentBase> uuuidTeam =  getAgentService().save(teamsToAdd);
1046
1047         for (UUID u:uuuidTeam.keySet()){
1048             titleCacheTeam.put(uuuidTeam.get(u).getTitleCache(), (Team) uuuidTeam.get(u) );
1049         }
1050
1051         state.getConfig().setTeams(titleCacheTeam);
1052         state.getConfig().setPersons(titleCachePerson);
1053     }
1054
1055     private boolean askQuestion(String question){
1056         Scanner scan = new Scanner(System.in);
1057         System.out.println(question);
1058         int index = scan.nextInt();
1059         if (index == 1) {
1060             return true;
1061         } else {
1062             return false;
1063         }
1064     }
1065
1066     @Override
1067     protected boolean doCheck(SpecimenSynthesysExcelImportState state) {
1068         logger.warn("Validation not yet implemented for " + getClass().getSimpleName());
1069         return true;
1070     }
1071 }