cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenSythesysExcelImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.excel.in;
  11
  12 import java.io.FileNotFoundException;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.util.ArrayList;
  16 import java.util.Calendar;
  17 import java.util.HashMap;
  18 import java.util.HashSet;
  19 import java.util.List;
  20 import java.util.Map;
  21 import java.util.Scanner;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.StringUtils;
  26 import org.apache.log4j.Logger;
  27 import org.springframework.stereotype.Component;
  28 import org.springframework.transaction.TransactionStatus;
  29
  30 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  31 import eu.etaxonomy.cdm.common.ExcelUtils;
  32 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  33 import eu.etaxonomy.cdm.io.common.ICdmIO;
  34 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  35 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  36 import eu.etaxonomy.cdm.model.agent.AgentBase;
  37 import eu.etaxonomy.cdm.model.agent.Institution;
  38 import eu.etaxonomy.cdm.model.agent.Person;
  39 import eu.etaxonomy.cdm.model.agent.Team;
  40 import eu.etaxonomy.cdm.model.common.CdmBase;
  41 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  42 import eu.etaxonomy.cdm.model.common.Language;
  43 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  44 import eu.etaxonomy.cdm.model.common.TimePeriod;
  45 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  46 import eu.etaxonomy.cdm.model.description.Feature;
  47 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  48 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  49 import eu.etaxonomy.cdm.model.location.NamedArea;
  50 import eu.etaxonomy.cdm.model.media.Media;
  51 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  52 import eu.etaxonomy.cdm.model.name.NonViralName;
  53 import eu.etaxonomy.cdm.model.occurrence.Collection;
  54 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  55 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  56 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  57 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  58 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  59 import eu.etaxonomy.cdm.model.reference.Reference;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  61 import eu.etaxonomy.cdm.model.taxon.Classification;
  62 import eu.etaxonomy.cdm.model.taxon.Taxon;
  63 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  64 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  65 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  66 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  67 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  68
  69 /**
  70  * @author p.kelbert
  71  * @created 29.10.2008
  72  * @date 13 mars 2013
  73  *
  74  */
  75 @Component
  76 public class SpecimenSythesysExcelImport  extends CdmImportBase<SpecimenSynthesysExcelImportConfigurator, SpecimenSynthesysExcelImportState>
  77         implements ICdmIO<SpecimenSynthesysExcelImportState> {
  78
  79     private static final Logger logger = Logger.getLogger(SpecimenSythesysExcelImport.class);
  80
  81     protected String fullScientificNameString;
  82     protected String nomenclatureCode;
  83     protected String institutionCode;
  84     protected String collectionCode;
  85     protected String unitID;
  86     protected String recordBasis;
  87     protected String accessionNumber;
  88     protected String fieldNumber;
  89     protected Double longitude;
  90     protected Double latitude;
  91     protected String locality;
  92     protected String languageIso = null;
  93     protected String country;
  94     protected String isocountry;
  95     protected int depth;
  96     protected int altitude;
  97     protected String gatheringYear;
  98     protected String gatheringMonth;
  99     protected String gatheringDay;
 100     protected String gatheringDate;
 101     protected String gatheringTeam;
 102     protected String gatheringAgent;
 103     protected String originalsource;
 104     protected String identifier;
 105     protected String gatheringNotes;
 106
 107     private DerivedUnit derivedUnitBase;
 108     private Reference<?> ref = null;
 109     private TransactionStatus tx;
 110     private Classification classification = null;
 111
 112     protected ArrayList<String> identificationList = new ArrayList<String>();
 113     protected ArrayList<String> multimediaObjects = new ArrayList<String>();
 114     private boolean keepAtomisedDate=true;
 115     private boolean useTDWGarea = true;
 116
 117     boolean DEBUG =false;
 118
 119      public SpecimenSythesysExcelImport() {
 120         super();
 121     }
 122
 123     /**
 124      * private HashMap that handle null values (missing keys)
 125      * return empty string instead of null
 126      * */
 127     public class MyHashMap<K,V> extends HashMap<K,V> {
 128         /**
 129          *
 130          */
 131         private static final long serialVersionUID = -6230407405666753405L;
 132
 133         @SuppressWarnings("unchecked")
 134         @Override
 135         public V get(Object key) {
 136             Object a;
 137             if (containsKey(key)) {
 138                 a= super.get(key);
 139             } else {
 140                 a="";
 141             }
 142             if (a ==null || a.toString().equalsIgnoreCase("none")) {
 143                 a="";
 144             }
 145             return (V) a;
 146         }
 147     }
 148
 149
 150     /**
 151      * getClassification : get the classification declared in the ImportState
 152      *
 153      * @param state
 154      * @return
 155      */
 156     private void setClassification(SpecimenSynthesysExcelImportState state) {
 157         if (classification == null){
 158             boolean configreused=false;
 159             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Goosefoots")) {
 160                 classification = getClassificationService().find(UUID.fromString("2c2dc41c-9891-42cd-9cd5-8b28dfdd1b8a"));
 161                 if (classification ==null){
 162                     String name = state.getConfig().getClassificationName();
 163                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 164                     getClassificationService().saveOrUpdate(classification);
 165                 }
 166                 configreused=true;
 167             }
 168             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Ants")) {
 169                 classification = getClassificationService().find(UUID.fromString("9b72d32f-2868-4f0d-81bd-ebc38ee8c645"));
 170                 if (classification ==null){
 171                     String name = state.getConfig().getClassificationName();
 172                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 173                     getClassificationService().saveOrUpdate(classification);
 174                 }
 175                 configreused=true;
 176             }
 177             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Fungi")) {
 178                 classification = getClassificationService().find(UUID.fromString("fe07274d-daff-4678-9f7d-2c3916948c3e"));
 179                 if (classification ==null){
 180                     String name = state.getConfig().getClassificationName();
 181                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 182                     getClassificationService().saveOrUpdate(classification);
 183                 }
 184                 configreused=true;
 185             }
 186             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Campylopus")) {
 187                 classification = getClassificationService().find(UUID.fromString("9dccbd11-eb0e-4cac-ae04-aa61ad82b8a6"));
 188                 if (classification ==null){
 189                     String name = state.getConfig().getClassificationName();
 190                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 191                     getClassificationService().saveOrUpdate(classification);
 192                 }
 193                 configreused=true;
 194             }
 195
 196             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Eupolybothrus")) {
 197                 classification = getClassificationService().find(UUID.fromString("62ce65e0-e036-49aa-aae7-4d4d0f88966c"));
 198                 if (classification ==null){
 199                     String name = state.getConfig().getClassificationName();
 200                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 201                     getClassificationService().saveOrUpdate(classification);
 202                 }
 203                 configreused=true;
 204             }
 205             if(!configreused){
 206
 207                 String name = state.getConfig().getClassificationName();
 208                 classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 209                 if (state.getConfig().getClassificationUuid() != null) {
 210                     classification.setUuid(state.getConfig().getClassificationUuid());
 211                 }
 212                 getClassificationService().saveOrUpdate(classification);
 213             }
 214
 215         }
 216     }
 217
 218     /**
 219      * refresh the hibernate transaction :
 220      * - commit the current queries
 221      * - get the reference and the classification and the derivedUnitBase back from the hibernate session
 222      * */
 223     private void refreshTransaction(){
 224         commitTransaction(tx);
 225         tx = startTransaction();
 226         ref = getReferenceService().find(ref.getUuid());
 227         classification = getClassificationService().find(classification.getUuid());
 228         try{
 229             derivedUnitBase = (DerivedUnit) getOccurrenceService().find(derivedUnitBase.getUuid());
 230         }catch(Exception e){
 231             //logger.warn("derivedunit up to date or not created yet");
 232         }
 233     }
 234
 235     //    /**
 236     //     * refresh the hibernate transaction :
 237     //     * - commit the current queries
 238     //     * - get the reference and the classification and the derivedUnitBase back from the hibernate session
 239     //     * */
 240     //    private void refreshTransac(){
 241     //        commitTransaction(tx);
 242     //        tx = startTransaction();
 243     //    }
 244     /*
 245      * Store the unit's properties into variables
 246      * @param unit: the hashmap containing the splitted Excel line (Key=column name, value=value)
 247      */
 248     private void setUnitPropertiesExcel(MyHashMap<String,String> unit, String defaultAuthor){
 249         multimediaObjects = new ArrayList<String>();
 250         identificationList = new ArrayList<String>();
 251         gatheringTeam="";
 252         gatheringAgent="";
 253
 254         String author = unit.get("author");
 255         if (author.isEmpty() && !defaultAuthor.isEmpty()) {
 256             author=defaultAuthor;
 257         }
 258         String taxonName = unit.get("taxonName");
 259
 260         institutionCode = unit.get("institution");
 261         collectionCode = unit.get("collection");
 262         unitID = unit.get("unitID");
 263         recordBasis = unit.get("recordBasis");
 264
 265         accessionNumber = null;
 266
 267         try {longitude = Double.valueOf(unit.get("longitude"));
 268         } catch (Exception e) {longitude = 0.0;}
 269
 270         try {latitude = Double.valueOf(unit.get("latitude"));
 271         } catch (Exception e) {latitude = 0.0;}
 272
 273         country = unit.get("country");
 274         isocountry = unit.get("isoCountry");
 275         locality = unit.get("locality");
 276         String habitat = unit.get("habitat");
 277         String stateProvince = unit.get("stateProvince");
 278         if (!stateProvince.isEmpty()) {
 279             locality=stateProvince+", "+locality;
 280         }
 281         if (!habitat.isEmpty()){
 282             locality+=" (Habitat: "+habitat+")";
 283         }
 284
 285         fieldNumber = unit.get("field number");
 286
 287         String url =unit.get("url");
 288         if (!url.isEmpty()) {
 289             multimediaObjects.add(url);
 290         }
 291
 292         String coll =unit.get("collector");
 293         if (!coll.isEmpty()) {
 294             if (coll.indexOf("& al.")!=-1 || coll.indexOf("et al.") != -1 || coll.indexOf(" al. ")!=-1 ||
 295                     coll.indexOf("&") != -1 || coll.indexOf(" et ") != -1 || coll.indexOf(";")!=-1) {
 296                 gatheringTeam = coll;
 297             } else{
 298                 //single
 299                 gatheringAgent = coll;
 300             }
 301         }
 302
 303         if(!taxonName.contains(author)) {
 304             identificationList.add(taxonName+" "+author);
 305         } else {
 306             identificationList.add(taxonName);
 307         }
 308
 309         gatheringYear = unit.get("year");
 310         gatheringMonth = unit.get("month");
 311         gatheringDay = unit.get("day");
 312         gatheringDate = unit.get("date");
 313
 314         gatheringNotes = unit.get("eventRemarks");
 315         originalsource = unit.get("originalsource");
 316         identifier = unit.get("identifier");
 317     }
 318
 319
 320     private Institution getInstitution(SpecimenSynthesysExcelImportConfigurator config){
 321         Institution institution;
 322         List<Institution> institutions;
 323         try{
 324             institutions= getAgentService().searchInstitutionByCode(institutionCode);
 325         }catch(Exception e){
 326             institutions=new ArrayList<Institution>();
 327         }
 328         if (institutions.size() ==0 || !config.getReUseExistingMetadata()){
 329             logger.debug("Institution (agent) unknown or not allowed to reuse existing metadata");
 330             //create institution
 331             institution = Institution.NewInstance();
 332             institution.setCode(institutionCode);
 333             getAgentService().saveOrUpdate(institution);
 334         }
 335         else{
 336             logger.debug("Institution (agent) already in the db");
 337             institution = institutions.get(0);
 338         }
 339         return CdmBase.deproxy(institution, Institution.class);
 340     }
 341
 342     /*
 343      * Look if the Collection does already exists
 344      * @param collectionCode: a string
 345      * @param institution: the current Institution
 346      * @param app
 347      * @return the Collection (existing or new)
 348      */
 349     private Collection getCollection(Institution institution, SpecimenSynthesysExcelImportConfigurator config){
 350         Collection collection =null;
 351         List<Collection> collections;
 352         try{
 353             collections = getCollectionService().searchByCode(collectionCode);
 354         }catch(Exception e){
 355             collections=new ArrayList<Collection>();
 356         }
 357         if (collections.size() ==0 || !config.getReUseExistingMetadata()){
 358             logger.debug("Collection not found or do not reuse existing metadata  "+collectionCode);
 359             //create new collection
 360             collection= Collection.NewInstance();
 361             collection.setCode(collectionCode);
 362             collection.setCodeStandard("GBIF");
 363             collection.setInstitute(institution);
 364         }
 365         else{
 366             boolean collectionFound=false;
 367             for (int i=0; i<collections.size(); i++){
 368                 collection = collections.get(i);
 369                 try {
 370                     if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())){
 371                         //found a collection with the same code and the same institution
 372                         collectionFound=true;
 373                     }
 374                 } catch (NullPointerException e) {}
 375             }
 376             if (!collectionFound){
 377                 collection= Collection.NewInstance();
 378                 collection.setCode(collectionCode);
 379                 collection.setCodeStandard("GBIF");
 380                 collection.setInstitute(institution);
 381             }
 382
 383         }
 384         return collection;
 385     }
 386
 387     /*
 388      *
 389      * @param app
 390      * @param derivedThing
 391      * @param sec
 392      */
 393     private void setTaxonNameBase(SpecimenSynthesysExcelImportConfigurator config){
 394         NonViralName<?> taxonName = null;
 395         Taxon taxon = null;
 396
 397         String scientificName="";
 398         boolean preferredFlag=false;
 399
 400         for (int i = 0; i < identificationList.size(); i++) {
 401             String fullScientificNameString = identificationList.get(i);
 402             fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
 403             if (fullScientificNameString.indexOf("_preferred_") != -1){
 404                 scientificName = fullScientificNameString.split("_preferred_")[0].trim();
 405                 String pTmp = fullScientificNameString.split("_preferred_")[1].split("_code_")[0];
 406                 if (pTmp == "1" || pTmp.toLowerCase().indexOf("true") != -1) {
 407                     preferredFlag=true;
 408                 } else {
 409                     preferredFlag=false;
 410                 }
 411             }else {
 412                 scientificName = fullScientificNameString.trim();
 413             }
 414
 415             if (fullScientificNameString.indexOf("_code_") != -1){
 416                 nomenclatureCode = fullScientificNameString.split("_code_")[1].trim();
 417             }
 418
 419             if (config.getDoReUseTaxon()){
 420                 List<TaxonBase> c = null;
 421                 try {
 422                     Taxon cc= getTaxonService().findBestMatchingTaxon(scientificName);
 423                     if (cc != null){
 424                         if ((cc.getSec() == null || cc.getSec().toString().isEmpty()) || (cc.getSec() != null &&
 425                                 cc.getSec().getTitleCache().equalsIgnoreCase(ref.getTitleCache()))) {
 426                             if(cc.getSec() == null || cc.getSec().toString().isEmpty()){
 427                                 cc.setSec(ref);
 428                                 getTaxonService().saveOrUpdate(cc);
 429                             }
 430                             taxon=cc;
 431                         }
 432                     }
 433                     else{
 434                         c = getTaxonService().searchTaxaByName(scientificName, ref);
 435                         for (TaxonBase<?> b : c) {
 436                             taxon = (Taxon) b;
 437                         }
 438                     }
 439                 } catch (Exception e) {
 440                     logger.info("Searchtaxabyname failed" + e);
 441                     taxon = null;
 442                 }
 443
 444             }
 445             if (!config.getDoReUseTaxon() || taxon == null){
 446                 if (DEBUG) {
 447                     logger.info("create new taxonName instance");
 448                 }
 449                 if (config.getDoAutomaticParsing()){
 450                     taxonName = parseScientificName(scientificName);
 451                 }
 452                 else{
 453                     taxonName = NonViralName.NewInstance(null);
 454                     taxonName.setTitleCache(scientificName, true);
 455                 }
 456                 getNameService().save(taxonName);
 457                 taxon = Taxon.NewInstance(taxonName, ref); //sec set null
 458                 getTaxonService().save(taxon);
 459                 //   refreshTransaction();
 460                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 461                 taxon = addTaxonNode(taxon, config);
 462             }
 463
 464
 465
 466             DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 467             determinationEvent.setTaxon(getTaxonService().find(taxon.getUuid()));
 468             determinationEvent.setPreferredFlag(preferredFlag);
 469
 470             determinationEvent.setIdentifiedUnit(derivedUnitBase);
 471             if (!identifier.isEmpty()){
 472                 Person p = Person.NewInstance();
 473                 p.setTitleCache(identifier, true);
 474                 determinationEvent.setActor(p);
 475             }
 476             derivedUnitBase.addDetermination(determinationEvent);
 477
 478             makeIndividualsAssociation(taxon,determinationEvent,config);
 479
 480             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 481         }
 482
 483     }
 484
 485     /**
 486      * @param taxon
 487      * @param taxonName
 488      * @param config
 489      * @return
 490      */
 491     private Taxon addTaxonNode(Taxon taxon, SpecimenSynthesysExcelImportConfigurator config) {
 492         if (DEBUG) {
 493             logger.info("link taxon to a taxonNode");
 494         }
 495         boolean exist = false;
 496         for (TaxonNode p : classification.getAllNodes()){
 497             if(p.getTaxon().equals(taxon)) {
 498                 exist =true;
 499             }
 500         }
 501         if (!exist){
 502             taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 503             classification.addChildTaxon(taxon, ref, null);
 504             getClassificationService().saveOrUpdate(classification);
 505             //            refreshTransaction();
 506         }
 507         return (Taxon) getTaxonService().find(taxon.getUuid());
 508     }
 509
 510     private NonViralName<?> parseScientificName(String scientificName){
 511         if (DEBUG) {
 512             logger.debug("in parseScientificName");
 513         }
 514         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 515         NonViralName<?>taxonName = null;
 516         boolean problem=false;
 517
 518         if (DEBUG) {
 519             logger.debug("nomenclature: "+nomenclatureCode);
 520         }
 521
 522         if(nomenclatureCode == null){
 523             taxonName = NonViralName.NewInstance(null);
 524             taxonName.setTitleCache(scientificName, true);
 525             return taxonName;
 526         }
 527
 528         if (nomenclatureCode.toString().equals("Zoological")){
 529             taxonName = nvnpi.parseFullName(scientificName,NomenclaturalCode.ICZN,null);
 530             if (taxonName.hasProblem()) {
 531                 problem=true;
 532             }
 533         }
 534         if (nomenclatureCode.toString().equals("Botanical")){
 535             taxonName  = nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNAFP,null);
 536             if (taxonName.hasProblem()) {
 537                 problem=true;
 538             }}
 539         if (nomenclatureCode.toString().equals("Bacterial")){
 540             taxonName = nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNB, null);
 541             if (taxonName.hasProblem()) {
 542                 problem=true;
 543             }
 544         }
 545         if (nomenclatureCode.toString().equals("Cultivar")){
 546             taxonName = nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNCP, null);
 547             if (taxonName.hasProblem()) {
 548                 problem=true;
 549             }
 550         }
 551         //      if (nomenclatureCode.toString().equals("Viral")){
 552         //      ViralName taxonName = (ViralName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICVCN(), null);
 553         //      if (taxonName.hasProblem())
 554         //      System.out.println("pb ICVCN");
 555         //      }
 556         //TODO: parsing of ViralNames?
 557         if(problem){
 558             taxonName = NonViralName.NewInstance(null);
 559             taxonName.setTitleCache(scientificName, true);
 560         }
 561         return taxonName;
 562
 563     }
 564
 565     private DerivedUnitFacade getFacade() {
 566         // logger.info("GETFACADE");
 567         /**
 568          * SPECIMEN OR OBSERVATION OR LIVING
 569          */
 570         // DerivedUnit derivedThing = null;
 571         SpecimenOrObservationType type = null;
 572
 573         // create specimen
 574         if (recordBasis != null) {
 575             String rec = recordBasis.toLowerCase();
 576             if (rec.contains("specimen") || rec.startsWith("s")) {// specimen
 577                 type = SpecimenOrObservationType.PreservedSpecimen;
 578             }
 579             if (rec.contains("observat") || rec.startsWith("o")) {
 580                 type = SpecimenOrObservationType.Observation;
 581             }
 582             if (rec.contains("fossil") || rec.startsWith("f") ){
 583                 type = SpecimenOrObservationType.Fossil;
 584             }
 585
 586             if (rec.contains("living") || rec.startsWith("l")) {
 587                 type = SpecimenOrObservationType.LivingSpecimen;
 588             }
 589             if (type == null) {
 590                 if(DEBUG) {
 591                     logger.info("The basis of record does not seem to be known: "   + recordBasis);
 592                 }
 593                 type = SpecimenOrObservationType.DerivedUnit;
 594             }
 595         } else {
 596             logger.info("The basis of record is null");
 597             type = SpecimenOrObservationType.DerivedUnit;
 598         }
 599         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 600         return derivedUnitFacade;
 601     }
 602
 603     /*
 604      * Store the unit with it's Gathering informations in the CDM
 605      */
 606     public boolean start(SpecimenSynthesysExcelImportConfigurator config){
 607         boolean result = true;
 608
 609         //        refreshTransaction();
 610         try {
 611
 612             /**
 613              * SPECIMEN OR OBSERVATION OR LIVING
 614              */
 615             DerivedUnitFacade derivedUnitFacade = getFacade();
 616             derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 617
 618             //set catalogue number (unitID)
 619             derivedUnitFacade.setCatalogNumber(unitID);
 620             derivedUnitFacade.setAccessionNumber(accessionNumber);
 621
 622             if (!originalsource.isEmpty()){
 623                 Reference<?> reference = ReferenceFactory.newGeneric();
 624                 reference.setTitleCache(originalsource, true);
 625                 derivedUnitBase.addSource(OriginalSourceType.Import, originalsource, "", reference, "");
 626             }else{
 627                 derivedUnitBase.addSource(OriginalSourceType.Import, unitID,"",ref,ref.getCitation());
 628             }
 629             /**
 630              * INSTITUTION & COLLECTION
 631              */
 632             //manage institution
 633             if (!institutionCode.isEmpty()){
 634                 Institution institution = getInstitution(config);
 635                 //manage collection
 636                 if (!collectionCode.isEmpty()){
 637                     Collection collection = getCollection(institution, config);
 638                     //link specimen & collection
 639                     derivedUnitFacade.setCollection(collection);
 640                 }
 641             }
 642
 643             /**
 644              * GATHERING EVENT
 645              */
 646             // gathering event
 647             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(getTermService(), locality, languageIso,
 648                     longitude, latitude, gatheringAgent, gatheringTeam,config);
 649             // country
 650             UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea();
 651             unitsGatheringArea.useTDWGareas(this.useTDWGarea);
 652             //            unitsGatheringArea.setConfig(config, getOccurrenceService(),getTermService());
 653             unitsGatheringArea.setParams(isocountry, country, config, getTermService(), getOccurrenceService());
 654             DefinedTermBase areaCountry =unitsGatheringArea.getCountry();
 655
 656
 657
 658             // copy gathering event to facade
 659             GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 660             //join gatheringEvent to fieldObservation
 661             derivedUnitFacade.setGatheringEvent(gatheringEvent);
 662
 663             derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 664             derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 665             //derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 666             derivedUnitFacade.setCountry((NamedArea)areaCountry);
 667             for(DefinedTermBase<?> area:unitsGatheringArea.getAreas()){
 668                 derivedUnitFacade.addCollectingArea((NamedArea) area);
 669             }
 670             if (gatheringNotes != null && !gatheringNotes.isEmpty()) {
 671                 derivedUnitFacade.setFieldNotes(gatheringNotes);
 672             }
 673
 674
 675             /*
 676              * merge AND STORE DATA
 677              */
 678             //            getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 679             getTermService().saveLanguageData(unitsGatheringEvent.getLocality());
 680
 681
 682
 683             //add gathering date
 684             if (keepAtomisedDate && (!gatheringYear.isEmpty() || !gatheringMonth.isEmpty() || !gatheringDay.isEmpty())){
 685                 Calendar calendar =  Calendar.getInstance();
 686                 if (!gatheringYear.isEmpty()) {
 687                     TimePeriod tp = TimePeriod.NewInstance(Integer.parseInt(gatheringYear));
 688                     if (!gatheringMonth.isEmpty()) {
 689                         tp.setStartMonth(Integer.parseInt(gatheringMonth));
 690                         if (!gatheringDay.isEmpty()) {
 691                             tp.setStartDay(Integer.parseInt(gatheringDay));
 692                         }
 693                     }
 694                     unitsGatheringEvent.setGatheringDate(tp);
 695                 }
 696
 697             }else{
 698                 if (!gatheringDate.isEmpty()){
 699                     TimePeriod tp = TimePeriodParser.parseString(gatheringDate);
 700                     unitsGatheringEvent.setGatheringDate(tp);
 701                 }
 702             }
 703
 704
 705             //add fieldNumber
 706             derivedUnitFacade.setFieldNumber(fieldNumber);
 707
 708             //add Multimedia URLs
 709             if(multimediaObjects.size()>0){
 710                 for (String multimediaObject : multimediaObjects) {
 711                     Media media;
 712                     try {
 713                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA);
 714                         derivedUnitFacade.addDerivedUnitMedia(media);
 715                     } catch (MalformedURLException e) {
 716                         // TODO Auto-generated catch block
 717                         e.printStackTrace();
 718                     }
 719                 }
 720             }
 721
 722             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 723
 724             setTaxonNameBase(config);
 725
 726             //         refreshTransaction();
 727             if (DEBUG) {
 728                 logger.info("saved new specimen ...");
 729             }
 730
 731         } catch (Exception e) {
 732             logger.warn("Error when reading record!!");
 733             e.printStackTrace();
 734             result = false;
 735         }
 736         if (DEBUG) {
 737             logger.info("commit done");
 738         }
 739         //app.close();
 740         return result;
 741     }
 742
 743     /*
 744      * Store the unit with it's Gathering informations in the CDM
 745      */
 746     public boolean resetCollectionInstitution(SpecimenSynthesysExcelImportConfigurator config,List<SpecimenOrObservationBase> specimenOrObs){
 747         boolean result = true;
 748
 749
 750         for (SpecimenOrObservationBase<?> specimen:specimenOrObs){
 751             DerivedUnit derivedUnit=null;
 752             if (specimen.isInstanceOf(GatheringEvent.class)){
 753                 System.out.println("gathering");
 754                 GatheringEvent gath = CdmBase.deproxy(specimen, GatheringEvent.class);
 755             }
 756             if (specimen.isInstanceOf(DerivedUnit.class)){
 757                 derivedUnit = CdmBase.deproxy(specimen, DerivedUnit.class);
 758                 String unitIDDB = derivedUnit.getCatalogNumber();
 759                 if (unitIDDB !=null) {
 760                     if(unitIDDB.equalsIgnoreCase(this.unitID)){
 761                         System.out.println("unitID found");
 762
 763                         Institution institution = getInstitution(config);
 764
 765                         Collection col = getCollection(institution, config);
 766                         getCollectionService().saveOrUpdate(col);
 767                         derivedUnit.setCollection(col);
 768                         getOccurrenceService().saveOrUpdate(derivedUnit);
 769                         return result;
 770                     }
 771                 }
 772             }
 773             if(specimen.isInstanceOf(IndividualsAssociation.class)) {
 774                 System.out.println("Indivi assoc");
 775             }
 776         }
 777         return result;
 778     }
 779
 780
 781     private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
 782         if (unit == null){
 783             return null;
 784         }
 785         SpecimenOrObservationType type = unit.getRecordBasis();
 786
 787         if (type.isFeatureObservation()){
 788             return Feature.OBSERVATION();
 789         }else if (type.isPreservedSpecimen() ||
 790                 type == SpecimenOrObservationType.LivingSpecimen ||
 791                 type == SpecimenOrObservationType.OtherSpecimen
 792                 ){
 793             return Feature.SPECIMEN();
 794         }else if (type == SpecimenOrObservationType.Unknown ||
 795                 type == SpecimenOrObservationType.DerivedUnit
 796                 ) {
 797             return Feature.INDIVIDUALS_ASSOCIATION();
 798         }
 799         if (DEBUG) {
 800             logger.warn("No feature defined for derived unit class: "
 801                     + unit.getClass().getSimpleName());
 802         }
 803         return null;
 804     }
 805
 806     private void makeIndividualsAssociation(Taxon taxon, DeterminationEvent determinationEvent, SpecimenSynthesysExcelImportConfigurator config) {
 807         //        try{
 808         //            taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 809         //        }
 810         //        catch(Exception e){
 811         //            //logger.info("taxon uptodate");
 812         //        }
 813
 814         TaxonDescription taxonDescription=null;
 815         if (taxon.getTitleCache().contains("Chenopodium vulvaria L.")) {
 816             taxonDescription = (TaxonDescription) getDescriptionService().find(UUID.fromString("a60074e7-979b-41fd-ad4d-d391db474ac4"));
 817         }
 818         if(!taxon.getTitleCache().contains("Chenopodium vulvaria L.") || taxonDescription ==null) {
 819             taxonDescription = getTaxonDescription(taxon, ref, false, true);
 820             taxonDescription.setTitleCache(ref.getTitleCache(), true);
 821         }
 822
 823
 824         taxon.addDescription(taxonDescription);
 825
 826         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
 827         Feature feature = makeFeature(derivedUnitBase);
 828         indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
 829         indAssociation.setFeature(feature);
 830
 831         for (Reference<?> citation : determinationEvent.getReferences()) {
 832             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
 833         }
 834         indAssociation.addSource(OriginalSourceType.Import, null,null,config.getDataReference(),null);
 835
 836         taxonDescription.addElement(indAssociation);
 837
 838         getDescriptionService().saveOrUpdate(taxonDescription);
 839         getTaxonService().saveOrUpdate(taxon);
 840     }
 841
 842     @Override
 843     protected boolean isIgnore(SpecimenSynthesysExcelImportState state) {
 844         return false;
 845     }
 846
 847     @Override
 848     protected void doInvoke(SpecimenSynthesysExcelImportState state) {
 849         boolean success = true;
 850         if (state.getConfig().doAskForDate()) {
 851             keepAtomisedDate = askQuestion("Gathering dates can be stored in either atomised fieds (day month year) or in a concatenated field."+
 852                     "\nWhich value do you want to store?\nPress 1 for the atomised, press 2 for the concatenated field, and then press enter.");
 853             useTDWGarea = askQuestion("Use TDWG area or Country/namedarea?\n Press 1 for TDWG areas\n Press 2 for classics Country.");
 854         }
 855
 856         tx = startTransaction();
 857
 858         ref = state.getConfig().getTaxonReference();
 859         if (ref == null){
 860             ref = state.getConfig().getSourceReference();
 861         }
 862         AgentBase<?> agent= ref.getAuthorship();
 863         if (agent != null){
 864             if (agent.getClass().equals(Team.class)){
 865                 for (Person p : ((Team) agent).getTeamMembers()){
 866                     getAgentService().saveOrUpdate(p);
 867                 }
 868             }
 869             if (agent.getClass().equals(Person.class)){
 870                 getAgentService().saveOrUpdate(agent);
 871             }
 872         }
 873         getReferenceService().saveOrUpdate(ref);
 874         setClassification(state);
 875
 876         refreshTransaction();
 877
 878         URI source = state.getConfig().getSource();
 879         ArrayList<HashMap<String,String>> unitsList = null;
 880         try{
 881             unitsList = ExcelUtils.parseXLS(source);
 882             logger.info("unitslist : "+unitsList.size());
 883         } catch(FileNotFoundException e){
 884             String message = "File not found: " + source;
 885             warnProgress(state, message, e);
 886             logger.error(message);
 887         }
 888
 889         if (unitsList != null){
 890             //load collectors in the database
 891             if (!state.getConfig().isDebugInstitutionOnly()) {
 892                 prepareCollectors(unitsList,state);
 893             }
 894
 895             List<SpecimenOrObservationBase> specimenOrObs = null;
 896             if (state.getConfig().isDebugInstitutionOnly()){
 897                 //DEBUG CHENOPODIUM VULVARIA
 898                 UUID uuid = UUID.fromString("85234ff5-8e40-4813-8f06-44ab960a905a");
 899                 Taxon taxon = (Taxon)getTaxonService().find(uuid);
 900
 901                 specimenOrObs = getOccurrenceService().listByAssociatedTaxon(null, null, taxon, null, null, null, null, null);
 902             }
 903             HashMap<String,String> unit=null;
 904             MyHashMap<String,String> myunit;
 905             for (int i=0; i<unitsList.size();i++){
 906                 //            for (int i=0; i<10;i++){
 907                 if (i%20==0) {
 908                     logger.info("NbUnit prepared: "+i);
 909                     refreshTransaction();
 910                 }
 911                 unit = unitsList.get(i);
 912                 myunit=new MyHashMap<String, String>();
 913                 for (String key :unit.keySet()) {
 914                     myunit.put(key, unit.get(key));
 915                 }
 916                 System.out.println(myunit.toString());
 917                 //FIXME do this via state
 918                 setUnitPropertiesExcel(myunit, state.getConfig().getDefaultAuthor());//and then invoke
 919                 if (state.getConfig().isDebugInstitutionOnly()){
 920                     success &= resetCollectionInstitution(state.getConfig(),specimenOrObs);
 921                 } else {
 922                     success &= start(state.getConfig());
 923                 }
 924             }
 925         }
 926         if (success == false){
 927             state.setUnsuccessfull();
 928         }
 929         commitTransaction(tx);
 930         return;
 931     }
 932
 933
 934     /**
 935      * @param unitsList
 936      * @param state
 937      */
 938     private void prepareCollectors(ArrayList<HashMap<String, String>> unitsList, SpecimenSynthesysExcelImportState state) {
 939         System.out.println("PREPARE COLLECTORS");
 940         List<String> collectors = new ArrayList<String>();
 941         List<String> teams = new ArrayList<String>();
 942         List<List<String>> collectorinteams = new ArrayList<List<String>>();
 943         String tmp;
 944         for (HashMap<String,String> unit : unitsList){
 945             tmp=null;
 946             tmp = unit.get("collector");
 947             if (tmp != null && !tmp.isEmpty()) {
 948                 if (tmp.indexOf("et al.") != -1 || tmp.indexOf(" al. ")!=-1 || tmp.indexOf("& al.")!=-1  ) {
 949                     if (!tmp.trim().isEmpty()) {
 950                         teams.add(tmp.trim());
 951                     }
 952                 } else{
 953                     if(tmp.indexOf(" et ")!=-1 || tmp.indexOf("&")!=-1 || tmp.indexOf(";") !=-1){
 954                         List<String> collteam = new ArrayList<String>();
 955                         String[] tmp1 = tmp.split(" et ");
 956                         for (String elt:tmp1){
 957                             String tmp2[] = elt.split("&");
 958                             for (String elt2:tmp2) {
 959                                 if (!elt2.trim().isEmpty()) {
 960                                     String tmp3[] = elt.split(";");
 961                                     for (String elt3:tmp3) {
 962                                         if (!elt3.isEmpty()){
 963                                             collectors.add(elt3.trim());
 964                                             collteam.add(elt3.trim());
 965                                         }
 966                                     }
 967                                 }
 968                             }
 969                         }
 970                         if (collteam.size()>0) {
 971                             collectorinteams.add(new ArrayList<String>(new HashSet<String>(collteam)));
 972                         }
 973                     }
 974                     else
 975                         if (!tmp.trim().isEmpty() && !tmp.toString().trim().equalsIgnoreCase("none")) {
 976                             collectors.add(tmp.trim());
 977                         }
 978                 }
 979             }
 980         }
 981
 982         List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
 983         List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
 984
 985
 986         //existing teams in DB
 987         Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
 988         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
 989
 990         Set<UUID> uuids = new HashSet<UUID>();
 991         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
 992             uuids.add(hibernateT.getUuid());
 993         }
 994         if (!uuids.isEmpty()){
 995             List<AgentBase> existingTeams = getAgentService().find(uuids);
 996             for (AgentBase<?> existingP:existingTeams){
 997                 titleCacheTeam.put(existingP.getTitleCache(),(Team) existingP);
 998             }
 999         }
1000
1001
1002         Map<String,UUID> teamMap = new HashMap<String, UUID>();
1003         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
1004             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
1005         }
1006
1007         //existing persons in DB
1008         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
1009         Map<String,Person> titleCachePerson = new HashMap<String, Person>();
1010         uuids = new HashSet<UUID>();
1011         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
1012             uuids.add(hibernateP.getUuid());
1013         }
1014
1015         if (!uuids.isEmpty()){
1016             List<AgentBase> existingPersons = getAgentService().find(uuids);
1017             for (AgentBase<?> existingP:existingPersons){
1018                 titleCachePerson.put(existingP.getTitleCache(),CdmBase.deproxy(existingP, Person.class));
1019             }
1020         }
1021
1022         Map<String,UUID> personMap = new HashMap<String, UUID>();
1023         for (UuidAndTitleCache<Person> person:hiberPersons){
1024             personMap.put(person.getTitleCache(), person.getUuid());
1025         }
1026
1027         java.util.Collection<AgentBase> personsToAdd = new ArrayList<AgentBase>();
1028         java.util.Collection<AgentBase> teamsToAdd = new ArrayList<AgentBase>();
1029
1030         for (String collector:collectorsU){
1031             Person p = Person.NewInstance();
1032             p.setTitleCache(collector,true);
1033             if (!personMap.containsKey(p.getTitleCache())){
1034                 personsToAdd.add(p);
1035             }
1036         }
1037         for (String team:teamsU){
1038             Team p = Team.NewInstance();
1039             p.setTitleCache(team,true);
1040             if (!teamMap.containsKey(p.getTitleCache())){
1041                 teamsToAdd.add(p);
1042             }
1043         }
1044         Map<UUID, AgentBase> uuuidPerson = getAgentService().save(personsToAdd);
1045         for (UUID u:uuuidPerson.keySet()){
1046             titleCachePerson.put(uuuidPerson.get(u).getTitleCache(),CdmBase.deproxy(uuuidPerson.get(u), Person.class) );
1047         }
1048
1049
1050         Person ptmp ;
1051         Map <String,Integer>teamdone = new HashMap<String, Integer>();
1052         for (List<String> collteam: collectorinteams){
1053             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
1054                 Team team = new Team();
1055                 boolean em =true;
1056                 for (String collector:collteam){
1057                     ptmp = Person.NewInstance();
1058                     ptmp.setTitleCache(collector,true);
1059                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
1060                     team.addTeamMember(p2);
1061                     em=false;
1062                 }
1063                 if (!em) {
1064                     teamsToAdd.add(team);
1065                 }
1066                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
1067             }
1068         }
1069
1070
1071         Map<UUID, AgentBase> uuuidTeam =  getAgentService().save(teamsToAdd);
1072
1073         for (UUID u:uuuidTeam.keySet()){
1074             titleCacheTeam.put(uuuidTeam.get(u).getTitleCache(), (Team) uuuidTeam.get(u) );
1075         }
1076
1077         state.getConfig().setTeams(titleCacheTeam);
1078         state.getConfig().setPersons(titleCachePerson);
1079     }
1080
1081     private boolean askQuestion(String question){
1082         Scanner scan = new Scanner(System.in);
1083         System.out.println(question);
1084         int index = scan.nextInt();
1085         if (index == 1) {
1086             return true;
1087         } else {
1088             return false;
1089         }
1090     }
1091
1092
1093     @Override
1094     protected boolean doCheck(SpecimenSynthesysExcelImportState state) {
1095         logger.warn("Validation not yet implemented for " + getClass().getSimpleName());
1096         return true;
1097     }
1098 }