cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenSythesysExcelImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.excel.in;
  11
  12 import java.io.FileNotFoundException;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.util.ArrayList;
  16 import java.util.Calendar;
  17 import java.util.HashMap;
  18 import java.util.HashSet;
  19 import java.util.List;
  20 import java.util.Map;
  21 import java.util.Scanner;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.StringUtils;
  26 import org.apache.log4j.Logger;
  27 import org.springframework.stereotype.Component;
  28 import org.springframework.transaction.TransactionStatus;
  29
  30 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  31 import eu.etaxonomy.cdm.common.ExcelUtils;
  32 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  33 import eu.etaxonomy.cdm.io.common.ICdmIO;
  34 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  35 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  36 import eu.etaxonomy.cdm.model.agent.AgentBase;
  37 import eu.etaxonomy.cdm.model.agent.Institution;
  38 import eu.etaxonomy.cdm.model.agent.Person;
  39 import eu.etaxonomy.cdm.model.agent.Team;
  40 import eu.etaxonomy.cdm.model.common.CdmBase;
  41 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  42 import eu.etaxonomy.cdm.model.common.Language;
  43 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  44 import eu.etaxonomy.cdm.model.common.TimePeriod;
  45 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  46 import eu.etaxonomy.cdm.model.description.Feature;
  47 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  48 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  49 import eu.etaxonomy.cdm.model.location.NamedArea;
  50 import eu.etaxonomy.cdm.model.media.Media;
  51 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  52 import eu.etaxonomy.cdm.model.name.TaxonName;
  53 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  54 import eu.etaxonomy.cdm.model.occurrence.Collection;
  55 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  56 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  57 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  58 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  59 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  60 import eu.etaxonomy.cdm.model.reference.Reference;
  61 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  62 import eu.etaxonomy.cdm.model.taxon.Classification;
  63 import eu.etaxonomy.cdm.model.taxon.Taxon;
  64 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  65 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  66 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  67 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  68 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  69
  70 /**
  71  * @author p.kelbert
  72  * @created 29.10.2008
  73  * @date 13 mars 2013
  74  *
  75  */
  76 @Component
  77 public class SpecimenSythesysExcelImport  extends CdmImportBase<SpecimenSynthesysExcelImportConfigurator, SpecimenSynthesysExcelImportState>
  78         implements ICdmIO<SpecimenSynthesysExcelImportState> {
  79
  80     private static final Logger logger = Logger.getLogger(SpecimenSythesysExcelImport.class);
  81
  82     protected String fullScientificNameString;
  83     protected String nomenclatureCode;
  84     protected String institutionCode;
  85     protected String collectionCode;
  86     protected String unitID;
  87     protected String recordBasis;
  88     protected String accessionNumber;
  89     protected String fieldNumber;
  90     protected Double longitude;
  91     protected Double latitude;
  92     protected String locality;
  93     protected String languageIso = null;
  94     protected String country;
  95     protected String isocountry;
  96     protected int depth;
  97     protected int altitude;
  98     protected String gatheringYear;
  99     protected String gatheringMonth;
 100     protected String gatheringDay;
 101     protected String gatheringDate;
 102     protected String gatheringTeam;
 103     protected String gatheringAgent;
 104     protected String originalsource;
 105     protected String identifier;
 106     protected String gatheringNotes;
 107
 108     private DerivedUnit derivedUnitBase;
 109     private Reference ref = null;
 110     private TransactionStatus tx;
 111     private Classification classification = null;
 112
 113     protected ArrayList<String> identificationList = new ArrayList<String>();
 114     protected ArrayList<String> multimediaObjects = new ArrayList<String>();
 115     private boolean keepAtomisedDate=true;
 116     private boolean useTDWGarea = true;
 117
 118     boolean DEBUG =false;
 119
 120      public SpecimenSythesysExcelImport() {
 121         super();
 122     }
 123
 124     /**
 125      * private HashMap that handle null values (missing keys)
 126      * return empty string instead of null
 127      * */
 128     public class MyHashMap<K,V> extends HashMap<K,V> {
 129         /**
 130          *
 131          */
 132         private static final long serialVersionUID = -6230407405666753405L;
 133
 134         @SuppressWarnings("unchecked")
 135         @Override
 136         public V get(Object key) {
 137             Object a;
 138             if (containsKey(key)) {
 139                 a= super.get(key);
 140             } else {
 141                 a="";
 142             }
 143             if (a ==null || a.toString().equalsIgnoreCase("none")) {
 144                 a="";
 145             }
 146             return (V) a;
 147         }
 148     }
 149
 150
 151     /**
 152      * getClassification : get the classification declared in the ImportState
 153      *
 154      * @param state
 155      * @return
 156      */
 157     private void setClassification(SpecimenSynthesysExcelImportState state) {
 158         if (classification == null){
 159             boolean configreused=false;
 160             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Goosefoots")) {
 161                 classification = getClassificationService().find(UUID.fromString("2c2dc41c-9891-42cd-9cd5-8b28dfdd1b8a"));
 162                 if (classification ==null){
 163                     String name = state.getConfig().getClassificationName();
 164                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 165                     getClassificationService().saveOrUpdate(classification);
 166                 }
 167                 configreused=true;
 168             }
 169             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Ants")) {
 170                 classification = getClassificationService().find(UUID.fromString("9b72d32f-2868-4f0d-81bd-ebc38ee8c645"));
 171                 if (classification ==null){
 172                     String name = state.getConfig().getClassificationName();
 173                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 174                     getClassificationService().saveOrUpdate(classification);
 175                 }
 176                 configreused=true;
 177             }
 178             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Fungi")) {
 179                 classification = getClassificationService().find(UUID.fromString("fe07274d-daff-4678-9f7d-2c3916948c3e"));
 180                 if (classification ==null){
 181                     String name = state.getConfig().getClassificationName();
 182                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 183                     getClassificationService().saveOrUpdate(classification);
 184                 }
 185                 configreused=true;
 186             }
 187             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Campylopus")) {
 188                 classification = getClassificationService().find(UUID.fromString("9dccbd11-eb0e-4cac-ae04-aa61ad82b8a6"));
 189                 if (classification ==null){
 190                     String name = state.getConfig().getClassificationName();
 191                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 192                     getClassificationService().saveOrUpdate(classification);
 193                 }
 194                 configreused=true;
 195             }
 196
 197             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("Eupolybothrus")) {
 198                 classification = getClassificationService().find(UUID.fromString("62ce65e0-e036-49aa-aae7-4d4d0f88966c"));
 199                 if (classification ==null){
 200                     String name = state.getConfig().getClassificationName();
 201                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 202                     getClassificationService().saveOrUpdate(classification);
 203                 }
 204                 configreused=true;
 205             }
 206             if(!configreused){
 207
 208                 String name = state.getConfig().getClassificationName();
 209                 classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 210                 if (state.getConfig().getClassificationUuid() != null) {
 211                     classification.setUuid(state.getConfig().getClassificationUuid());
 212                 }
 213                 getClassificationService().saveOrUpdate(classification);
 214             }
 215
 216         }
 217     }
 218
 219     /**
 220      * refresh the hibernate transaction :
 221      * - commit the current queries
 222      * - get the reference and the classification and the derivedUnitBase back from the hibernate session
 223      * */
 224     private void refreshTransaction(){
 225         commitTransaction(tx);
 226         tx = startTransaction();
 227         ref = getReferenceService().find(ref.getUuid());
 228         classification = getClassificationService().find(classification.getUuid());
 229         try{
 230             derivedUnitBase = (DerivedUnit) getOccurrenceService().find(derivedUnitBase.getUuid());
 231         }catch(Exception e){
 232             //logger.warn("derivedunit up to date or not created yet");
 233         }
 234     }
 235
 236     //    /**
 237     //     * refresh the hibernate transaction :
 238     //     * - commit the current queries
 239     //     * - get the reference and the classification and the derivedUnitBase back from the hibernate session
 240     //     * */
 241     //    private void refreshTransac(){
 242     //        commitTransaction(tx);
 243     //        tx = startTransaction();
 244     //    }
 245     /*
 246      * Store the unit's properties into variables
 247      * @param unit: the hashmap containing the splitted Excel line (Key=column name, value=value)
 248      */
 249     private void setUnitPropertiesExcel(MyHashMap<String,String> unit, String defaultAuthor){
 250         multimediaObjects = new ArrayList<String>();
 251         identificationList = new ArrayList<String>();
 252         gatheringTeam="";
 253         gatheringAgent="";
 254
 255         String author = unit.get("author");
 256         if (author.isEmpty() && !defaultAuthor.isEmpty()) {
 257             author=defaultAuthor;
 258         }
 259         String taxonName = unit.get("taxonName");
 260
 261         institutionCode = unit.get("institution");
 262         collectionCode = unit.get("collection");
 263         unitID = unit.get("unitID");
 264         recordBasis = unit.get("recordBasis");
 265
 266         accessionNumber = null;
 267
 268         try {longitude = Double.valueOf(unit.get("longitude"));
 269         } catch (Exception e) {longitude = 0.0;}
 270
 271         try {latitude = Double.valueOf(unit.get("latitude"));
 272         } catch (Exception e) {latitude = 0.0;}
 273
 274         country = unit.get("country");
 275         isocountry = unit.get("isoCountry");
 276         locality = unit.get("locality");
 277         String habitat = unit.get("habitat");
 278         String stateProvince = unit.get("stateProvince");
 279         if (!stateProvince.isEmpty()) {
 280             locality=stateProvince+", "+locality;
 281         }
 282         if (!habitat.isEmpty()){
 283             locality+=" (Habitat: "+habitat+")";
 284         }
 285
 286         fieldNumber = unit.get("field number");
 287
 288         String url =unit.get("url");
 289         if (!url.isEmpty()) {
 290             multimediaObjects.add(url);
 291         }
 292
 293         String coll =unit.get("collector");
 294         if (!coll.isEmpty()) {
 295             if (coll.indexOf("& al.")!=-1 || coll.indexOf("et al.") != -1 || coll.indexOf(" al. ")!=-1 ||
 296                     coll.indexOf("&") != -1 || coll.indexOf(" et ") != -1 || coll.indexOf(";")!=-1) {
 297                 gatheringTeam = coll;
 298             } else{
 299                 //single
 300                 gatheringAgent = coll;
 301             }
 302         }
 303
 304         if(!taxonName.contains(author)) {
 305             identificationList.add(taxonName+" "+author);
 306         } else {
 307             identificationList.add(taxonName);
 308         }
 309
 310         gatheringYear = unit.get("year");
 311         gatheringMonth = unit.get("month");
 312         gatheringDay = unit.get("day");
 313         gatheringDate = unit.get("date");
 314
 315         gatheringNotes = unit.get("eventRemarks");
 316         originalsource = unit.get("originalsource");
 317         identifier = unit.get("identifier");
 318     }
 319
 320
 321     private Institution getInstitution(SpecimenSynthesysExcelImportConfigurator config){
 322         Institution institution;
 323         List<Institution> institutions;
 324         try{
 325             institutions= getAgentService().searchInstitutionByCode(institutionCode);
 326         }catch(Exception e){
 327             institutions=new ArrayList<Institution>();
 328         }
 329         if (institutions.size() ==0 || !config.getReUseExistingMetadata()){
 330             logger.debug("Institution (agent) unknown or not allowed to reuse existing metadata");
 331             //create institution
 332             institution = Institution.NewInstance();
 333             institution.setCode(institutionCode);
 334             getAgentService().saveOrUpdate(institution);
 335         }
 336         else{
 337             logger.debug("Institution (agent) already in the db");
 338             institution = institutions.get(0);
 339         }
 340         return CdmBase.deproxy(institution, Institution.class);
 341     }
 342
 343     /*
 344      * Look if the Collection does already exists
 345      * @param collectionCode: a string
 346      * @param institution: the current Institution
 347      * @param app
 348      * @return the Collection (existing or new)
 349      */
 350     private Collection getCollection(Institution institution, SpecimenSynthesysExcelImportConfigurator config){
 351         Collection collection =null;
 352         List<Collection> collections;
 353         try{
 354             collections = getCollectionService().searchByCode(collectionCode);
 355         }catch(Exception e){
 356             collections=new ArrayList<Collection>();
 357         }
 358         if (collections.size() ==0 || !config.getReUseExistingMetadata()){
 359             logger.debug("Collection not found or do not reuse existing metadata  "+collectionCode);
 360             //create new collection
 361             collection= Collection.NewInstance();
 362             collection.setCode(collectionCode);
 363             collection.setCodeStandard("GBIF");
 364             collection.setInstitute(institution);
 365         }
 366         else{
 367             boolean collectionFound=false;
 368             for (int i=0; i<collections.size(); i++){
 369                 collection = collections.get(i);
 370                 try {
 371                     if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())){
 372                         //found a collection with the same code and the same institution
 373                         collectionFound=true;
 374                     }
 375                 } catch (NullPointerException e) {}
 376             }
 377             if (!collectionFound){
 378                 collection= Collection.NewInstance();
 379                 collection.setCode(collectionCode);
 380                 collection.setCodeStandard("GBIF");
 381                 collection.setInstitute(institution);
 382             }
 383
 384         }
 385         return collection;
 386     }
 387
 388     /*
 389      *
 390      * @param app
 391      * @param derivedThing
 392      * @param sec
 393      */
 394     private void setTaxonName(SpecimenSynthesysExcelImportConfigurator config){
 395         TaxonName taxonName = null;
 396         Taxon taxon = null;
 397
 398         String scientificName="";
 399         boolean preferredFlag=false;
 400
 401         for (int i = 0; i < identificationList.size(); i++) {
 402             String fullScientificNameString = identificationList.get(i);
 403             fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
 404             if (fullScientificNameString.indexOf("_preferred_") != -1){
 405                 scientificName = fullScientificNameString.split("_preferred_")[0].trim();
 406                 String pTmp = fullScientificNameString.split("_preferred_")[1].split("_code_")[0];
 407                 if (pTmp == "1" || pTmp.toLowerCase().indexOf("true") != -1) {
 408                     preferredFlag=true;
 409                 } else {
 410                     preferredFlag=false;
 411                 }
 412             }else {
 413                 scientificName = fullScientificNameString.trim();
 414             }
 415
 416             if (fullScientificNameString.indexOf("_code_") != -1){
 417                 nomenclatureCode = fullScientificNameString.split("_code_")[1].trim();
 418             }
 419
 420             if (config.getDoReUseTaxon()){
 421                 List<TaxonBase> c = null;
 422                 try {
 423                     Taxon cc= getTaxonService().findBestMatchingTaxon(scientificName);
 424                     if (cc != null){
 425                         if ((cc.getSec() == null || cc.getSec().toString().isEmpty()) || (cc.getSec() != null &&
 426                                 cc.getSec().getTitleCache().equalsIgnoreCase(ref.getTitleCache()))) {
 427                             if(cc.getSec() == null || cc.getSec().toString().isEmpty()){
 428                                 cc.setSec(ref);
 429                                 getTaxonService().saveOrUpdate(cc);
 430                             }
 431                             taxon=cc;
 432                         }
 433                     }
 434                     else{
 435                         c = getTaxonService().searchTaxaByName(scientificName, ref);
 436                         for (TaxonBase<?> b : c) {
 437                             taxon = (Taxon) b;
 438                         }
 439                     }
 440                 } catch (Exception e) {
 441                     logger.info("Searchtaxabyname failed" + e);
 442                     taxon = null;
 443                 }
 444
 445             }
 446             if (!config.getDoReUseTaxon() || taxon == null){
 447                 if (DEBUG) {
 448                     logger.info("create new taxonName instance");
 449                 }
 450                 if (config.getDoAutomaticParsing()){
 451                     taxonName = parseScientificName(scientificName);
 452                 }
 453                 else{
 454                     taxonName = TaxonNameFactory.NewNonViralInstance(null);
 455                     taxonName.setTitleCache(scientificName, true);
 456                 }
 457                 getNameService().save(taxonName);
 458                 taxon = Taxon.NewInstance(taxonName, ref); //sec set null
 459                 getTaxonService().save(taxon);
 460                 //   refreshTransaction();
 461                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 462                 taxon = addTaxonNode(taxon, config);
 463             }
 464
 465
 466
 467             DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 468             determinationEvent.setTaxon(getTaxonService().find(taxon.getUuid()));
 469             determinationEvent.setPreferredFlag(preferredFlag);
 470
 471             determinationEvent.setIdentifiedUnit(derivedUnitBase);
 472             if (!identifier.isEmpty()){
 473                 Person p = Person.NewInstance();
 474                 p.setTitleCache(identifier, true);
 475                 determinationEvent.setActor(p);
 476             }
 477             derivedUnitBase.addDetermination(determinationEvent);
 478
 479             makeIndividualsAssociation(taxon,determinationEvent,config);
 480
 481             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 482         }
 483
 484     }
 485
 486     /**
 487      * @param taxon
 488      * @param taxonName
 489      * @param config
 490      * @return
 491      */
 492     private Taxon addTaxonNode(Taxon taxon, SpecimenSynthesysExcelImportConfigurator config) {
 493         if (DEBUG) {
 494             logger.info("link taxon to a taxonNode");
 495         }
 496         boolean exist = false;
 497         for (TaxonNode p : classification.getAllNodes()){
 498             if(p.getTaxon().equals(taxon)) {
 499                 exist =true;
 500             }
 501         }
 502         if (!exist){
 503             taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 504             classification.addChildTaxon(taxon, ref, null);
 505             getClassificationService().saveOrUpdate(classification);
 506             //            refreshTransaction();
 507         }
 508         return (Taxon) getTaxonService().find(taxon.getUuid());
 509     }
 510
 511     private TaxonName parseScientificName(String scientificName){
 512         if (DEBUG) {
 513             logger.debug("in parseScientificName");
 514         }
 515         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 516         TaxonName taxonName = null;
 517         boolean problem=false;
 518
 519         if (DEBUG) {
 520             logger.debug("nomenclature: "+nomenclatureCode);
 521         }
 522
 523         if(nomenclatureCode == null){
 524             taxonName = TaxonNameFactory.NewNonViralInstance(null);
 525             taxonName.setTitleCache(scientificName, true);
 526             return taxonName;
 527         }
 528
 529         if (nomenclatureCode.toString().equals("Zoological")){
 530             taxonName = (TaxonName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICZN,null);
 531             if (taxonName.hasProblem()) {
 532                 problem=true;
 533             }
 534         }
 535         if (nomenclatureCode.toString().equals("Botanical")){
 536             taxonName  = (TaxonName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNAFP,null);
 537             if (taxonName.hasProblem()) {
 538                 problem=true;
 539             }}
 540         if (nomenclatureCode.toString().equals("Bacterial")){
 541             taxonName = (TaxonName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNB, null);
 542             if (taxonName.hasProblem()) {
 543                 problem=true;
 544             }
 545         }
 546         if (nomenclatureCode.toString().equals("Cultivar")){
 547             taxonName = (TaxonName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNCP, null);
 548             if (taxonName.hasProblem()) {
 549                 problem=true;
 550             }
 551         }
 552         //      if (nomenclatureCode.toString().equals("Viral")){
 553         //      ViralName taxonName = (ViralName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICVCN(), null);
 554         //      if (taxonName.hasProblem())
 555         //      System.out.println("pb ICVCN");
 556         //      }
 557         //TODO: parsing of ViralNames?
 558         if(problem){
 559             taxonName = TaxonNameFactory.NewNonViralInstance(null);
 560             taxonName.setTitleCache(scientificName, true);
 561         }
 562         return taxonName;
 563
 564     }
 565
 566     private DerivedUnitFacade getFacade() {
 567         // logger.info("GETFACADE");
 568         /**
 569          * SPECIMEN OR OBSERVATION OR LIVING
 570          */
 571         // DerivedUnit derivedThing = null;
 572         SpecimenOrObservationType type = null;
 573
 574         // create specimen
 575         if (recordBasis != null) {
 576             String rec = recordBasis.toLowerCase();
 577             if (rec.contains("specimen") || rec.startsWith("s")) {// specimen
 578                 type = SpecimenOrObservationType.PreservedSpecimen;
 579             }
 580             if (rec.contains("observat") || rec.startsWith("o")) {
 581                 type = SpecimenOrObservationType.Observation;
 582             }
 583             if (rec.contains("fossil") || rec.startsWith("f") ){
 584                 type = SpecimenOrObservationType.Fossil;
 585             }
 586
 587             if (rec.contains("living") || rec.startsWith("l")) {
 588                 type = SpecimenOrObservationType.LivingSpecimen;
 589             }
 590             if (type == null) {
 591                 if(DEBUG) {
 592                     logger.info("The basis of record does not seem to be known: "   + recordBasis);
 593                 }
 594                 type = SpecimenOrObservationType.DerivedUnit;
 595             }
 596         } else {
 597             logger.info("The basis of record is null");
 598             type = SpecimenOrObservationType.DerivedUnit;
 599         }
 600         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 601         return derivedUnitFacade;
 602     }
 603
 604     /*
 605      * Store the unit with it's Gathering informations in the CDM
 606      */
 607     public boolean start(SpecimenSynthesysExcelImportConfigurator config){
 608         boolean result = true;
 609
 610         //        refreshTransaction();
 611         try {
 612
 613             /**
 614              * SPECIMEN OR OBSERVATION OR LIVING
 615              */
 616             DerivedUnitFacade derivedUnitFacade = getFacade();
 617             derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 618
 619             //set catalogue number (unitID)
 620             derivedUnitFacade.setCatalogNumber(unitID);
 621             derivedUnitFacade.setAccessionNumber(accessionNumber);
 622
 623             if (!originalsource.isEmpty()){
 624                 Reference reference = ReferenceFactory.newGeneric();
 625                 reference.setTitleCache(originalsource, true);
 626                 derivedUnitBase.addSource(OriginalSourceType.Import, originalsource, "", reference, "");
 627             }else{
 628                 derivedUnitBase.addSource(OriginalSourceType.Import, unitID,"",ref,ref.getCitation());
 629             }
 630             /**
 631              * INSTITUTION & COLLECTION
 632              */
 633             //manage institution
 634             if (!institutionCode.isEmpty()){
 635                 Institution institution = getInstitution(config);
 636                 //manage collection
 637                 if (!collectionCode.isEmpty()){
 638                     Collection collection = getCollection(institution, config);
 639                     //link specimen & collection
 640                     derivedUnitFacade.setCollection(collection);
 641                 }
 642             }
 643
 644             /**
 645              * GATHERING EVENT
 646              */
 647             // gathering event
 648             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(getTermService(), locality, languageIso,
 649                     longitude, latitude, gatheringAgent, gatheringTeam,config);
 650             // country
 651             UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea();
 652             unitsGatheringArea.useTDWGareas(this.useTDWGarea);
 653             //            unitsGatheringArea.setConfig(config, getOccurrenceService(),getTermService());
 654             unitsGatheringArea.setParams(isocountry, country, config, getTermService(), getOccurrenceService());
 655             DefinedTermBase areaCountry =unitsGatheringArea.getCountry();
 656
 657
 658
 659             // copy gathering event to facade
 660             GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 661             //join gatheringEvent to fieldObservation
 662             derivedUnitFacade.setGatheringEvent(gatheringEvent);
 663
 664             derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 665             derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 666             //derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 667             derivedUnitFacade.setCountry((NamedArea)areaCountry);
 668             for(DefinedTermBase<?> area:unitsGatheringArea.getAreas()){
 669                 derivedUnitFacade.addCollectingArea((NamedArea) area);
 670             }
 671             if (gatheringNotes != null && !gatheringNotes.isEmpty()) {
 672                 derivedUnitFacade.setFieldNotes(gatheringNotes);
 673             }
 674
 675
 676             /*
 677              * merge AND STORE DATA
 678              */
 679             //            getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 680             getTermService().saveLanguageData(unitsGatheringEvent.getLocality());
 681
 682
 683
 684             //add gathering date
 685             if (keepAtomisedDate && (!gatheringYear.isEmpty() || !gatheringMonth.isEmpty() || !gatheringDay.isEmpty())){
 686                 Calendar calendar =  Calendar.getInstance();
 687                 if (!gatheringYear.isEmpty()) {
 688                     TimePeriod tp = TimePeriod.NewInstance(Integer.parseInt(gatheringYear));
 689                     if (!gatheringMonth.isEmpty()) {
 690                         tp.setStartMonth(Integer.parseInt(gatheringMonth));
 691                         if (!gatheringDay.isEmpty()) {
 692                             tp.setStartDay(Integer.parseInt(gatheringDay));
 693                         }
 694                     }
 695                     unitsGatheringEvent.setGatheringDate(tp);
 696                 }
 697
 698             }else{
 699                 if (!gatheringDate.isEmpty()){
 700                     TimePeriod tp = TimePeriodParser.parseString(gatheringDate);
 701                     unitsGatheringEvent.setGatheringDate(tp);
 702                 }
 703             }
 704
 705
 706             //add fieldNumber
 707             derivedUnitFacade.setFieldNumber(fieldNumber);
 708
 709             //add Multimedia URLs
 710             if(multimediaObjects.size()>0){
 711                 for (String multimediaObject : multimediaObjects) {
 712                     Media media;
 713                     try {
 714                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA);
 715                         derivedUnitFacade.addDerivedUnitMedia(media);
 716                     } catch (MalformedURLException e) {
 717                         // TODO Auto-generated catch block
 718                         e.printStackTrace();
 719                     }
 720                 }
 721             }
 722
 723             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 724
 725             setTaxonName(config);
 726
 727             //         refreshTransaction();
 728             if (DEBUG) {
 729                 logger.info("saved new specimen ...");
 730             }
 731
 732         } catch (Exception e) {
 733             logger.warn("Error when reading record!!");
 734             e.printStackTrace();
 735             result = false;
 736         }
 737         if (DEBUG) {
 738             logger.info("commit done");
 739         }
 740         //app.close();
 741         return result;
 742     }
 743
 744     /*
 745      * Store the unit with it's Gathering informations in the CDM
 746      */
 747     public boolean resetCollectionInstitution(SpecimenSynthesysExcelImportConfigurator config,List<SpecimenOrObservationBase> specimenOrObs){
 748         boolean result = true;
 749
 750
 751         for (SpecimenOrObservationBase<?> specimen:specimenOrObs){
 752             DerivedUnit derivedUnit=null;
 753             if (specimen.isInstanceOf(GatheringEvent.class)){
 754                 System.out.println("gathering");
 755                 GatheringEvent gath = CdmBase.deproxy(specimen, GatheringEvent.class);
 756             }
 757             if (specimen.isInstanceOf(DerivedUnit.class)){
 758                 derivedUnit = CdmBase.deproxy(specimen, DerivedUnit.class);
 759                 String unitIDDB = derivedUnit.getCatalogNumber();
 760                 if (unitIDDB !=null) {
 761                     if(unitIDDB.equalsIgnoreCase(this.unitID)){
 762                         System.out.println("unitID found");
 763
 764                         Institution institution = getInstitution(config);
 765
 766                         Collection col = getCollection(institution, config);
 767                         getCollectionService().saveOrUpdate(col);
 768                         derivedUnit.setCollection(col);
 769                         getOccurrenceService().saveOrUpdate(derivedUnit);
 770                         return result;
 771                     }
 772                 }
 773             }
 774             if(specimen.isInstanceOf(IndividualsAssociation.class)) {
 775                 System.out.println("Indivi assoc");
 776             }
 777         }
 778         return result;
 779     }
 780
 781
 782     private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
 783         if (unit == null){
 784             return null;
 785         }
 786         SpecimenOrObservationType type = unit.getRecordBasis();
 787
 788         if (type.isFeatureObservation()){
 789             return Feature.OBSERVATION();
 790         }else if (type.isPreservedSpecimen() ||
 791                 type == SpecimenOrObservationType.LivingSpecimen ||
 792                 type == SpecimenOrObservationType.OtherSpecimen
 793                 ){
 794             return Feature.SPECIMEN();
 795         }else if (type == SpecimenOrObservationType.Unknown ||
 796                 type == SpecimenOrObservationType.DerivedUnit
 797                 ) {
 798             return Feature.INDIVIDUALS_ASSOCIATION();
 799         }
 800         if (DEBUG) {
 801             logger.warn("No feature defined for derived unit class: "
 802                     + unit.getClass().getSimpleName());
 803         }
 804         return null;
 805     }
 806
 807     private void makeIndividualsAssociation(Taxon taxon, DeterminationEvent determinationEvent, SpecimenSynthesysExcelImportConfigurator config) {
 808         //        try{
 809         //            taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 810         //        }
 811         //        catch(Exception e){
 812         //            //logger.info("taxon uptodate");
 813         //        }
 814
 815         TaxonDescription taxonDescription=null;
 816         if (taxon.getTitleCache().contains("Chenopodium vulvaria L.")) {
 817             taxonDescription = (TaxonDescription) getDescriptionService().find(UUID.fromString("a60074e7-979b-41fd-ad4d-d391db474ac4"));
 818         }
 819         if(!taxon.getTitleCache().contains("Chenopodium vulvaria L.") || taxonDescription ==null) {
 820             taxonDescription = getTaxonDescription(taxon, ref, false, true);
 821             taxonDescription.setTitleCache(ref.getTitleCache(), true);
 822         }
 823
 824
 825         taxon.addDescription(taxonDescription);
 826
 827         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
 828         Feature feature = makeFeature(derivedUnitBase);
 829         indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
 830         indAssociation.setFeature(feature);
 831
 832         for (Reference citation : determinationEvent.getReferences()) {
 833             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
 834         }
 835         indAssociation.addSource(OriginalSourceType.Import, null,null,config.getDataReference(),null);
 836
 837         taxonDescription.addElement(indAssociation);
 838
 839         getDescriptionService().saveOrUpdate(taxonDescription);
 840         getTaxonService().saveOrUpdate(taxon);
 841     }
 842
 843     @Override
 844     protected boolean isIgnore(SpecimenSynthesysExcelImportState state) {
 845         return false;
 846     }
 847
 848     @Override
 849     protected void doInvoke(SpecimenSynthesysExcelImportState state) {
 850         boolean success = true;
 851         if (state.getConfig().doAskForDate()) {
 852             keepAtomisedDate = askQuestion("Gathering dates can be stored in either atomised fieds (day month year) or in a concatenated field."+
 853                     "\nWhich value do you want to store?\nPress 1 for the atomised, press 2 for the concatenated field, and then press enter.");
 854             useTDWGarea = askQuestion("Use TDWG area or Country/namedarea?\n Press 1 for TDWG areas\n Press 2 for classics Country.");
 855         }
 856
 857         tx = startTransaction();
 858
 859         ref = state.getConfig().getTaxonReference();
 860         if (ref == null){
 861             ref = state.getConfig().getSourceReference();
 862         }
 863         AgentBase<?> agent= ref.getAuthorship();
 864         if (agent != null){
 865             if (agent.getClass().equals(Team.class)){
 866                 for (Person p : ((Team) agent).getTeamMembers()){
 867                     getAgentService().saveOrUpdate(p);
 868                 }
 869             }
 870             if (agent.getClass().equals(Person.class)){
 871                 getAgentService().saveOrUpdate(agent);
 872             }
 873         }
 874         getReferenceService().saveOrUpdate(ref);
 875         setClassification(state);
 876
 877         refreshTransaction();
 878
 879         URI source = state.getConfig().getSource();
 880         ArrayList<HashMap<String,String>> unitsList = null;
 881         try{
 882             unitsList = ExcelUtils.parseXLS(source);
 883             logger.info("unitslist : "+unitsList.size());
 884         } catch(FileNotFoundException e){
 885             String message = "File not found: " + source;
 886             warnProgress(state, message, e);
 887             logger.error(message);
 888         }
 889
 890         if (unitsList != null){
 891             //load collectors in the database
 892             if (!state.getConfig().isDebugInstitutionOnly()) {
 893                 prepareCollectors(unitsList,state);
 894             }
 895
 896             List<SpecimenOrObservationBase> specimenOrObs = null;
 897             if (state.getConfig().isDebugInstitutionOnly()){
 898                 //DEBUG CHENOPODIUM VULVARIA
 899                 UUID uuid = UUID.fromString("85234ff5-8e40-4813-8f06-44ab960a905a");
 900                 Taxon taxon = (Taxon)getTaxonService().find(uuid);
 901
 902                 specimenOrObs = getOccurrenceService().listByAssociatedTaxon(null, null, taxon, null, null, null, null, null);
 903             }
 904             HashMap<String,String> unit=null;
 905             MyHashMap<String,String> myunit;
 906             for (int i=0; i<unitsList.size();i++){
 907                 //            for (int i=0; i<10;i++){
 908                 if (i%20==0) {
 909                     logger.info("NbUnit prepared: "+i);
 910                     refreshTransaction();
 911                 }
 912                 unit = unitsList.get(i);
 913                 myunit=new MyHashMap<String, String>();
 914                 for (String key :unit.keySet()) {
 915                     myunit.put(key, unit.get(key));
 916                 }
 917                 System.out.println(myunit.toString());
 918                 //FIXME do this via state
 919                 setUnitPropertiesExcel(myunit, state.getConfig().getDefaultAuthor());//and then invoke
 920                 if (state.getConfig().isDebugInstitutionOnly()){
 921                     success &= resetCollectionInstitution(state.getConfig(),specimenOrObs);
 922                 } else {
 923                     success &= start(state.getConfig());
 924                 }
 925             }
 926         }
 927         if (success == false){
 928             state.setUnsuccessfull();
 929         }
 930         commitTransaction(tx);
 931         return;
 932     }
 933
 934
 935     /**
 936      * @param unitsList
 937      * @param state
 938      */
 939     private void prepareCollectors(ArrayList<HashMap<String, String>> unitsList, SpecimenSynthesysExcelImportState state) {
 940         System.out.println("PREPARE COLLECTORS");
 941         List<String> collectors = new ArrayList<String>();
 942         List<String> teams = new ArrayList<String>();
 943         List<List<String>> collectorinteams = new ArrayList<List<String>>();
 944         String tmp;
 945         for (HashMap<String,String> unit : unitsList){
 946             tmp=null;
 947             tmp = unit.get("collector");
 948             if (tmp != null && !tmp.isEmpty()) {
 949                 if (tmp.indexOf("et al.") != -1 || tmp.indexOf(" al. ")!=-1 || tmp.indexOf("& al.")!=-1  ) {
 950                     if (!tmp.trim().isEmpty()) {
 951                         teams.add(tmp.trim());
 952                     }
 953                 } else{
 954                     if(tmp.indexOf(" et ")!=-1 || tmp.indexOf("&")!=-1 || tmp.indexOf(";") !=-1){
 955                         List<String> collteam = new ArrayList<String>();
 956                         String[] tmp1 = tmp.split(" et ");
 957                         for (String elt:tmp1){
 958                             String tmp2[] = elt.split("&");
 959                             for (String elt2:tmp2) {
 960                                 if (!elt2.trim().isEmpty()) {
 961                                     String tmp3[] = elt.split(";");
 962                                     for (String elt3:tmp3) {
 963                                         if (!elt3.isEmpty()){
 964                                             collectors.add(elt3.trim());
 965                                             collteam.add(elt3.trim());
 966                                         }
 967                                     }
 968                                 }
 969                             }
 970                         }
 971                         if (collteam.size()>0) {
 972                             collectorinteams.add(new ArrayList<String>(new HashSet<String>(collteam)));
 973                         }
 974                     }
 975                     else
 976                         if (!tmp.trim().isEmpty() && !tmp.toString().trim().equalsIgnoreCase("none")) {
 977                             collectors.add(tmp.trim());
 978                         }
 979                 }
 980             }
 981         }
 982
 983         List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
 984         List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
 985
 986
 987         //existing teams in DB
 988         Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
 989         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
 990
 991         Set<UUID> uuids = new HashSet<UUID>();
 992         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
 993             uuids.add(hibernateT.getUuid());
 994         }
 995         if (!uuids.isEmpty()){
 996             List<AgentBase> existingTeams = getAgentService().find(uuids);
 997             for (AgentBase<?> existingP:existingTeams){
 998                 titleCacheTeam.put(existingP.getTitleCache(),(Team) existingP);
 999             }
1000         }
1001
1002
1003         Map<String,UUID> teamMap = new HashMap<String, UUID>();
1004         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
1005             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
1006         }
1007
1008         //existing persons in DB
1009         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
1010         Map<String,Person> titleCachePerson = new HashMap<String, Person>();
1011         uuids = new HashSet<UUID>();
1012         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
1013             uuids.add(hibernateP.getUuid());
1014         }
1015
1016         if (!uuids.isEmpty()){
1017             List<AgentBase> existingPersons = getAgentService().find(uuids);
1018             for (AgentBase<?> existingP:existingPersons){
1019                 titleCachePerson.put(existingP.getTitleCache(),CdmBase.deproxy(existingP, Person.class));
1020             }
1021         }
1022
1023         Map<String,UUID> personMap = new HashMap<String, UUID>();
1024         for (UuidAndTitleCache<Person> person:hiberPersons){
1025             personMap.put(person.getTitleCache(), person.getUuid());
1026         }
1027
1028         java.util.Collection<AgentBase> personsToAdd = new ArrayList<AgentBase>();
1029         java.util.Collection<AgentBase> teamsToAdd = new ArrayList<AgentBase>();
1030
1031         for (String collector:collectorsU){
1032             Person p = Person.NewInstance();
1033             p.setTitleCache(collector,true);
1034             if (!personMap.containsKey(p.getTitleCache())){
1035                 personsToAdd.add(p);
1036             }
1037         }
1038         for (String team:teamsU){
1039             Team p = Team.NewInstance();
1040             p.setTitleCache(team,true);
1041             if (!teamMap.containsKey(p.getTitleCache())){
1042                 teamsToAdd.add(p);
1043             }
1044         }
1045         Map<UUID, AgentBase> uuuidPerson = getAgentService().save(personsToAdd);
1046         for (UUID u:uuuidPerson.keySet()){
1047             titleCachePerson.put(uuuidPerson.get(u).getTitleCache(),CdmBase.deproxy(uuuidPerson.get(u), Person.class) );
1048         }
1049
1050
1051         Person ptmp ;
1052         Map <String,Integer>teamdone = new HashMap<String, Integer>();
1053         for (List<String> collteam: collectorinteams){
1054             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
1055                 Team team = new Team();
1056                 boolean em =true;
1057                 for (String collector:collteam){
1058                     ptmp = Person.NewInstance();
1059                     ptmp.setTitleCache(collector,true);
1060                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
1061                     team.addTeamMember(p2);
1062                     em=false;
1063                 }
1064                 if (!em) {
1065                     teamsToAdd.add(team);
1066                 }
1067                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
1068             }
1069         }
1070
1071
1072         Map<UUID, AgentBase> uuuidTeam =  getAgentService().save(teamsToAdd);
1073
1074         for (UUID u:uuuidTeam.keySet()){
1075             titleCacheTeam.put(uuuidTeam.get(u).getTitleCache(), (Team) uuuidTeam.get(u) );
1076         }
1077
1078         state.getConfig().setTeams(titleCacheTeam);
1079         state.getConfig().setPersons(titleCachePerson);
1080     }
1081
1082     private boolean askQuestion(String question){
1083         Scanner scan = new Scanner(System.in);
1084         System.out.println(question);
1085         int index = scan.nextInt();
1086         if (index == 1) {
1087             return true;
1088         } else {
1089             return false;
1090         }
1091     }
1092
1093
1094     @Override
1095     protected boolean doCheck(SpecimenSynthesysExcelImportState state) {
1096         logger.warn("Validation not yet implemented for " + getClass().getSimpleName());
1097         return true;
1098     }
1099 }