cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenSythesysExcelImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.excel.in;
  11
  12 import java.io.FileNotFoundException;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.util.ArrayList;
  16 import java.util.Calendar;
  17 import java.util.HashMap;
  18 import java.util.HashSet;
  19 import java.util.List;
  20 import java.util.Map;
  21 import java.util.Scanner;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.StringUtils;
  26 import org.apache.log4j.Logger;
  27 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
  28 import org.springframework.stereotype.Component;
  29 import org.springframework.transaction.TransactionStatus;
  30
  31 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  32 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  33 import eu.etaxonomy.cdm.common.ExcelUtils;
  34 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  35 import eu.etaxonomy.cdm.io.common.ICdmIO;
  36 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  37 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  38 import eu.etaxonomy.cdm.model.agent.AgentBase;
  39 import eu.etaxonomy.cdm.model.agent.Institution;
  40 import eu.etaxonomy.cdm.model.agent.Person;
  41 import eu.etaxonomy.cdm.model.agent.Team;
  42 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  43 import eu.etaxonomy.cdm.model.common.Language;
  44 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  45 import eu.etaxonomy.cdm.model.common.TimePeriod;
  46 import eu.etaxonomy.cdm.model.common.UuidAndTitleCache;
  47 import eu.etaxonomy.cdm.model.description.Feature;
  48 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  49 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  50 import eu.etaxonomy.cdm.model.location.NamedArea;
  51 import eu.etaxonomy.cdm.model.media.Media;
  52 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  53 import eu.etaxonomy.cdm.model.name.NonViralName;
  54 import eu.etaxonomy.cdm.model.name.Rank;
  55 import eu.etaxonomy.cdm.model.occurrence.Collection;
  56 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  57 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
  58 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  59 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
  60 import eu.etaxonomy.cdm.model.occurrence.Fossil;
  61 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  62 import eu.etaxonomy.cdm.model.occurrence.LivingBeing;
  63 import eu.etaxonomy.cdm.model.occurrence.Observation;
  64 import eu.etaxonomy.cdm.model.occurrence.Specimen;
  65 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  66 import eu.etaxonomy.cdm.model.reference.Reference;
  67 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  68 import eu.etaxonomy.cdm.model.taxon.Classification;
  69 import eu.etaxonomy.cdm.model.taxon.Taxon;
  70 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  71 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  72 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  73
  74 /**
  75  * @author p.kelbert
  76  * @created 29.10.2008
  77  * @version 1.0
  78  *
  79  * @author pkelbert
  80  * @date 13 mars 2013
  81  *
  82  */
  83 @Component
  84 public class SpecimenSythesysExcelImport  extends CdmImportBase<SpecimenSynthesysExcelImportConfigurator, SpecimenSynthesysExcelImportState>  implements ICdmIO<SpecimenSynthesysExcelImportState> {
  85
  86     private static final Logger logger = Logger.getLogger(SpecimenSythesysExcelImport.class);
  87
  88     protected String fullScientificNameString;
  89     protected String nomenclatureCode;
  90     protected String institutionCode;
  91     protected String collectionCode;
  92     protected String unitID;
  93     protected String recordBasis;
  94     protected String accessionNumber;
  95     protected String fieldNumber;
  96     protected Double longitude;
  97     protected Double latitude;
  98     protected String locality;
  99     protected String languageIso = null;
 100     protected String country;
 101     protected String isocountry;
 102     protected int depth;
 103     protected int altitude;
 104     protected String gatheringYear;
 105     protected String gatheringMonth;
 106     protected String gatheringDay;
 107     protected String gatheringDate;
 108     protected String gatheringTeam;
 109     protected String gatheringAgent;
 110     protected String originalsource;
 111
 112     private DerivedUnitBase derivedUnitBase;
 113     private Reference<?> ref = null;
 114     private TransactionStatus tx;
 115     private Classification classification = null;
 116
 117     protected ArrayList<String> identificationList = new ArrayList<String>();
 118     protected ArrayList<String> multimediaObjects = new ArrayList<String>();
 119     private boolean keepAtomisedDate=true;
 120
 121     boolean DEBUG =false;
 122
 123     protected HSSFWorkbook hssfworkbook = null;
 124
 125     public SpecimenSythesysExcelImport() {
 126         super();
 127     }
 128
 129     /**
 130      * private HashMap that handle null values (missing keys)
 131      * return empty string instead of null
 132      * */
 133     public class MyHashMap<K,V> extends HashMap<K,V> {
 134         private static final long serialVersionUID = -6230407405666753405L;
 135
 136         @SuppressWarnings("unchecked")
 137         @Override
 138         public V get(Object key) {
 139             Object a;
 140             if (containsKey(key)) {
 141                 a= super.get(key);
 142             } else {
 143                 a="";
 144             }
 145             if (a ==null || a.toString().equalsIgnoreCase("none")) {
 146                 a="";
 147             }
 148             return (V) a;
 149         }
 150     }
 151
 152
 153     /**
 154      * getClassification : get the classification declared in the ImportState
 155      *
 156      * @param state
 157      * @return
 158      */
 159     private void setClassification(SpecimenSynthesysExcelImportState state) {
 160         if (classification == null){
 161             if (state.getConfig().getClassificationName()!=null && state.getConfig().getClassificationName().equalsIgnoreCase("chenopodium")) {
 162                 classification = getClassificationService().find(UUID.fromString("2c2dc41c-9891-42cd-9cd5-8b28dfdd1b8a"));
 163                 if (classification ==null){
 164                     String name = state.getConfig().getClassificationName();
 165                     classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 166                     getClassificationService().saveOrUpdate(classification);
 167                 }
 168             }
 169             else{
 170                 String name = state.getConfig().getClassificationName();
 171                 classification = Classification.NewInstance(name, ref, Language.DEFAULT());
 172                 if (state.getConfig().getClassificationUuid() != null) {
 173                     classification.setUuid(state.getConfig().getClassificationUuid());
 174                 }
 175                 getClassificationService().saveOrUpdate(classification);
 176             }
 177
 178         }
 179     }
 180
 181     /**
 182      * refresh the hibernate transaction :
 183      * - commit the current queries
 184      * - get the reference and the classification and the derivedUnitBase back from the hibernate session
 185      * */
 186     private void refreshTransaction(){
 187         commitTransaction(tx);
 188         tx = startTransaction();
 189         ref = getReferenceService().find(ref.getUuid());
 190         classification = getClassificationService().find(classification.getUuid());
 191         if (derivedUnitBase != null){
 192                 derivedUnitBase = (DerivedUnitBase) getOccurrenceService().find(derivedUnitBase.getUuid());
 193         }
 194     }
 195
 196     /*
 197      * Store the unit's properties into variables
 198      * @param unit: the hashmap containing the splitted Excel line (Key=column name, value=value)
 199      */
 200     private void setUnitPropertiesExcel(MyHashMap<String,String> unit, String defaultAuthor){
 201         multimediaObjects = new ArrayList<String>();
 202         identificationList = new ArrayList<String>();
 203         gatheringTeam="";
 204         gatheringAgent="";
 205
 206         String author = unit.get("author");
 207         if (author.isEmpty() && !defaultAuthor.isEmpty()) {
 208             author=defaultAuthor;
 209         }
 210         String taxonName = unit.get("taxonName");
 211
 212         institutionCode = unit.get("institution");
 213         collectionCode = unit.get("collection");
 214         unitID = unit.get("unitID");
 215         recordBasis = unit.get("recordBasis");
 216
 217         accessionNumber = null;
 218
 219         try {longitude = Double.valueOf(unit.get("longitude"));
 220         } catch (Exception e) {longitude = 0.0;}
 221
 222         try {latitude = Double.valueOf(unit.get("latitude"));
 223         } catch (Exception e) {latitude = 0.0;}
 224
 225         country = unit.get("country");
 226         isocountry = unit.get("isoCountry");
 227         locality = unit.get("locality");
 228
 229         fieldNumber = unit.get("field number");
 230
 231         String url =unit.get("url");
 232         if (!url.isEmpty()) {
 233             multimediaObjects.add(url);
 234         }
 235
 236         String coll =unit.get("collector");
 237         if (!coll.isEmpty()) {
 238             if (coll.indexOf("& al.")!=-1 || coll.indexOf("et al.") != -1 || coll.indexOf(" al. ")!=-1 || coll.indexOf("&") != -1 || coll.indexOf(" et ") != -1) {
 239                 gatheringTeam = coll;
 240             } else{
 241                 //single
 242                 gatheringAgent = coll;
 243             }
 244         }
 245
 246         identificationList.add(taxonName+" "+author);
 247
 248         gatheringYear = unit.get("year");
 249         gatheringMonth = unit.get("month");
 250         gatheringDay = unit.get("day");
 251         gatheringDate = unit.get("date");
 252
 253         originalsource = unit.get("originalsource");
 254     }
 255
 256
 257     private Institution getInstitution(SpecimenSynthesysExcelImportConfigurator config){
 258         Institution institution;
 259         List<Institution> institutions;
 260         try{
 261             institutions= getAgentService().searchInstitutionByCode(institutionCode);
 262         }catch(Exception e){
 263             institutions=new ArrayList<Institution>();
 264         }
 265         if (institutions.size() ==0 || !config.getReUseExistingMetadata()){
 266             logger.debug("Institution (agent) unknown or not allowed to reuse existing metadata");
 267             //create institution
 268             institution = Institution.NewInstance();
 269             institution.setCode(institutionCode);
 270         }
 271         else{
 272             logger.debug("Institution (agent) already in the db");
 273             institution = institutions.get(0);
 274         }
 275         return institution;
 276     }
 277
 278     /*
 279      * Look if the Collection does already exists
 280      * @param collectionCode: a string
 281      * @param institution: the current Institution
 282      * @param app
 283      * @return the Collection (existing or new)
 284      */
 285     private Collection getCollection(Institution institution, SpecimenSynthesysExcelImportConfigurator config){
 286         Collection collection = Collection.NewInstance();
 287         List<Collection> collections;
 288         try{
 289             collections = getCollectionService().searchByCode(collectionCode);
 290         }catch(Exception e){
 291             collections=new ArrayList<Collection>();
 292         }
 293         if (collections.size() ==0 || !config.getReUseExistingMetadata()){
 294             logger.debug("Collection not found or do not reuse existing metadata  "+collectionCode);
 295             //create new collection
 296             collection.setCode(collectionCode);
 297             collection.setCodeStandard("GBIF");
 298             collection.setInstitute(institution);
 299         }
 300         else{
 301             boolean collectionFound=false;
 302             for (int i=0; i<collections.size(); i++){
 303                 collection = collections.get(i);
 304                 try {
 305                     if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())){
 306                         //found a collection with the same code and the same institution
 307                         collectionFound=true;
 308                     }
 309                 } catch (NullPointerException e) {}
 310             }
 311             if (!collectionFound){
 312                 collection.setCode(collectionCode);
 313                 collection.setCodeStandard("GBIF");
 314                 collection.setInstitute(institution);
 315             }
 316
 317         }
 318         return collection;
 319     }
 320
 321     /*
 322      *
 323      * @param app
 324      * @param derivedThing
 325      * @param sec
 326      */
 327     private void setTaxonNameBase(SpecimenSynthesysExcelImportConfigurator config){
 328         NonViralName<?> taxonName = null;
 329         Taxon taxon = null;
 330
 331         String scientificName="";
 332         boolean preferredFlag=false;
 333
 334         for (int i = 0; i < identificationList.size(); i++) {
 335             String fullScientificNameString = identificationList.get(i);
 336             fullScientificNameString = fullScientificNameString.replaceAll(" et ", " & ");
 337             if (fullScientificNameString.indexOf("_preferred_") != -1){
 338                 scientificName = fullScientificNameString.split("_preferred_")[0].trim();
 339                 String pTmp = fullScientificNameString.split("_preferred_")[1].split("_code_")[0];
 340                 if (pTmp == "1" || pTmp.toLowerCase().indexOf("true") != -1) {
 341                     preferredFlag=true;
 342                 } else {
 343                     preferredFlag=false;
 344                 }
 345             }else {
 346                 scientificName = fullScientificNameString.trim();
 347             }
 348
 349             if (fullScientificNameString.indexOf("_code_") != -1){
 350                 nomenclatureCode = fullScientificNameString.split("_code_")[1].trim();
 351             }
 352
 353             if (config.getDoReUseTaxon()){
 354                 List<TaxonBase> c = null;
 355                 try {
 356                     Taxon cc= getTaxonService().findBestMatchingTaxon(scientificName);
 357                     if (cc != null){
 358                         if ((cc.getSec() == null || cc.getSec().toString().isEmpty()) || (cc.getSec() != null && cc.getSec().getTitleCache().equalsIgnoreCase(ref.getTitleCache()))) {
 359                             if(cc.getSec() == null || cc.getSec().toString().isEmpty()){
 360                                 cc.setSec(ref);
 361                                 getTaxonService().saveOrUpdate(cc);
 362                             }
 363                             taxon=cc;
 364                         }
 365                     }
 366                     else{
 367                         c = getTaxonService().searchTaxaByName(scientificName, ref);
 368                         for (TaxonBase b : c) {
 369                             taxon = (Taxon) b;
 370                         }
 371                     }
 372                 } catch (Exception e) {
 373                     logger.info("Searchtaxabyname failed" + e);
 374                     taxon = null;
 375                 }
 376
 377             }
 378             if (!config.getDoReUseTaxon() || taxon == null){
 379                 if (DEBUG) {
 380                     logger.info("create new taxonName instance");
 381                 }
 382                 if (config.getDoAutomaticParsing()){
 383                     taxonName = parseScientificName(scientificName);
 384                 }
 385                 else{
 386                     taxonName = NonViralName.NewInstance(Rank.UNKNOWN_RANK());
 387                     taxonName.setTitleCache(scientificName, true);
 388                 }
 389                 getNameService().save(taxonName);
 390                 taxon = Taxon.NewInstance(taxonName, ref); //sec set null
 391                 getTaxonService().save(taxon);
 392                 //   refreshTransaction();
 393                 taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 394                 taxon = addTaxonNode(taxon, config);
 395             }
 396
 397
 398
 399             DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 400             determinationEvent.setTaxon(getTaxonService().find(taxon.getUuid()));
 401             determinationEvent.setPreferredFlag(preferredFlag);
 402
 403             determinationEvent.setIdentifiedUnit(derivedUnitBase);
 404
 405             derivedUnitBase.addDetermination(determinationEvent);
 406
 407             makeIndividualsAssociation(taxon,determinationEvent);
 408
 409             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 410         }
 411
 412     }
 413
 414     /**
 415      * @param taxon
 416      * @param taxonName
 417      * @param config
 418      * @return
 419      */
 420     private Taxon addTaxonNode(Taxon taxon, SpecimenSynthesysExcelImportConfigurator config) {
 421         if (DEBUG) {
 422             logger.info("link taxon to a taxonNode");
 423         }
 424         boolean exist = false;
 425         for (TaxonNode p : classification.getAllNodes()){
 426             if(p.getTaxon().equals(taxon)) {
 427                 exist =true;
 428             }
 429         }
 430         if (!exist){
 431             taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 432             classification.addChildTaxon(taxon, ref, "", null);
 433             getClassificationService().saveOrUpdate(classification);
 434             //            refreshTransaction();
 435         }
 436         return (Taxon) getTaxonService().find(taxon.getUuid());
 437     }
 438
 439     private NonViralName<?> parseScientificName(String scientificName){
 440         if (DEBUG) {
 441             logger.debug("in parseScientificName");
 442         }
 443         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 444         NonViralName<?>taxonName = null;
 445         boolean problem=false;
 446
 447         if (DEBUG) {
 448             logger.debug("nomenclature: "+nomenclatureCode);
 449         }
 450
 451         if(nomenclatureCode == null){
 452             taxonName = NonViralName.NewInstance(Rank.UNKNOWN_RANK());
 453             taxonName.setTitleCache(scientificName, true);
 454             return taxonName;
 455         }
 456
 457         if (nomenclatureCode.toString().equals("Zoological")){
 458             taxonName = nvnpi.parseFullName(scientificName,NomenclaturalCode.ICZN,null);
 459             if (taxonName.hasProblem()) {
 460                 problem=true;
 461             }
 462         }
 463         if (nomenclatureCode.toString().equals("Botanical")){
 464             taxonName  = nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNAFP,null);
 465             if (taxonName.hasProblem()) {
 466                 problem=true;
 467             }}
 468         if (nomenclatureCode.toString().equals("Bacterial")){
 469             taxonName = nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNB, null);
 470             if (taxonName.hasProblem()) {
 471                 problem=true;
 472             }
 473         }
 474         if (nomenclatureCode.toString().equals("Cultivar")){
 475             taxonName = nvnpi.parseFullName(scientificName,NomenclaturalCode.ICNCP, null);
 476             if (taxonName.hasProblem()) {
 477                 problem=true;
 478             }
 479         }
 480         //      if (nomenclatureCode.toString().equals("Viral")){
 481         //      ViralName taxonName = (ViralName)nvnpi.parseFullName(scientificName,NomenclaturalCode.ICVCN(), null);
 482         //      if (taxonName.hasProblem())
 483         //      System.out.println("pb ICVCN");
 484         //      }
 485         //TODO: parsing of ViralNames?
 486         if(problem){
 487             taxonName = NonViralName.NewInstance(Rank.UNKNOWN_RANK());
 488             taxonName.setTitleCache(scientificName, true);
 489         }
 490         return taxonName;
 491
 492     }
 493
 494     private DerivedUnitFacade getFacade() {
 495         // logger.info("GETFACADE");
 496         /**
 497          * SPECIMEN OR OBSERVATION OR LIVING
 498          */
 499         // DerivedUnitBase derivedThing = null;
 500         DerivedUnitType type = null;
 501
 502         // create specimen
 503         if (recordBasis != null) {
 504             String rec = recordBasis.toLowerCase();
 505             if (rec.contains("specimen") || rec.startsWith("s")) {// specimen
 506                 type = DerivedUnitType.Specimen;
 507             }
 508             if (rec.contains("observat") || rec.startsWith("o")) {
 509                 type = DerivedUnitType.Observation;
 510             }
 511             if (rec.contains("fossil") || rec.startsWith("f") ){
 512                 type = DerivedUnitType.Fossil;
 513             }
 514
 515             if (rec.contains("living") || rec.startsWith("l")) {
 516                 type = DerivedUnitType.LivingBeing;
 517             }
 518             if (type == null) {
 519                 if(DEBUG) {
 520                     logger.info("The basis of record does not seem to be known: "   + recordBasis);
 521                 }
 522                 type = DerivedUnitType.DerivedUnit;
 523             }
 524         } else {
 525             logger.info("The basis of record is null");
 526             type = DerivedUnitType.DerivedUnit;
 527         }
 528         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 529         return derivedUnitFacade;
 530     }
 531
 532     /*
 533      * Store the unit with it's Gathering informations in the CDM
 534      */
 535     public boolean start(SpecimenSynthesysExcelImportConfigurator config){
 536         boolean result = true;
 537
 538         //        refreshTransaction();
 539         try {
 540
 541             /**
 542              * SPECIMEN OR OBSERVATION OR LIVING
 543              */
 544             DerivedUnitFacade derivedUnitFacade = getFacade();
 545             derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 546
 547             //set catalogue number (unitID)
 548             derivedUnitFacade.setCatalogNumber(unitID);
 549             derivedUnitFacade.setAccessionNumber(accessionNumber);
 550
 551             if (!originalsource.isEmpty()){
 552                 Reference<?> reference = ReferenceFactory.newGeneric();
 553                 reference.setTitleCache(originalsource, true);
 554                 derivedUnitBase.addSource(OriginalSourceType.Unknown, originalsource, "", reference, "");
 555             }
 556             /**
 557              * INSTITUTION & COLLECTION
 558              */
 559             //manage institution
 560             Institution institution = getInstitution(config);
 561             //manage collection
 562             Collection collection = getCollection(institution, config);
 563             //link specimen & collection
 564             derivedUnitFacade.setCollection(collection);
 565
 566             /**
 567              * GATHERING EVENT
 568              */
 569
 570             // gathering event
 571             UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(getTermService(), locality, languageIso,
 572                     longitude, latitude, gatheringAgent, gatheringTeam,config);
 573
 574             // country
 575             UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea(isocountry, country, this);
 576             NamedArea areaCountry = unitsGatheringArea.getArea();
 577
 578
 579
 580             // copy gathering event to facade
 581             GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 582
 583             //join gatheringEvent to fieldObservation
 584             derivedUnitFacade.setGatheringEvent(gatheringEvent);
 585
 586             derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 587             derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 588             //derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 589             derivedUnitFacade.setCountry(areaCountry);
 590             derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 591
 592
 593             /*
 594              * merge AND STORE DATA
 595              */
 596             getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 597             getTermService().saveLanguageData(unitsGatheringEvent.getLocality());
 598
 599
 600
 601             //add gathering date
 602             if (keepAtomisedDate && (!gatheringYear.isEmpty() || !gatheringMonth.isEmpty() || !gatheringDay.isEmpty())){
 603                 Calendar calendar =  Calendar.getInstance();
 604                 if (!gatheringYear.isEmpty()) {
 605                     TimePeriod tp = TimePeriod.NewInstance(Integer.parseInt(gatheringYear));
 606                     if (!gatheringMonth.isEmpty()) {
 607                         tp.setStartMonth(Integer.parseInt(gatheringMonth));
 608                         if (!gatheringDay.isEmpty()) {
 609                             tp.setStartDay(Integer.parseInt(gatheringDay));
 610                         }
 611                     }
 612                     unitsGatheringEvent.setGatheringDate(tp);
 613                 }
 614
 615             }else{
 616                 if (!gatheringDate.isEmpty()){
 617                     TimePeriod tp = TimePeriod.parseString(gatheringDate);
 618                     unitsGatheringEvent.setGatheringDate(tp);
 619                 }
 620             }
 621
 622
 623             //add fieldNumber
 624             derivedUnitFacade.setFieldNumber(NB(fieldNumber));
 625
 626             //add Multimedia URLs
 627             if(multimediaObjects.size()>0){
 628                 for (String multimediaObject : multimediaObjects) {
 629                     Media media;
 630                     try {
 631                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA, false);
 632                         derivedUnitFacade.addDerivedUnitMedia(media);
 633                     } catch (MalformedURLException e) {
 634                         // TODO Auto-generated catch block
 635                         e.printStackTrace();
 636                     }
 637                 }
 638             }
 639
 640             getOccurrenceService().saveOrUpdate(derivedUnitBase);
 641
 642             setTaxonNameBase(config);
 643
 644             //         refreshTransaction();
 645             if (DEBUG) {
 646                 logger.info("saved new specimen ...");
 647             }
 648
 649         } catch (Exception e) {
 650             logger.warn("Error when reading record!!");
 651             e.printStackTrace();
 652             result = false;
 653         }
 654         if (DEBUG) {
 655             logger.info("commit done");
 656         }
 657         //app.close();
 658         return result;
 659     }
 660
 661
 662     private Feature makeFeature(SpecimenOrObservationBase unit) {
 663         if (unit.isInstanceOf(DerivedUnit.class)) {
 664             return Feature.INDIVIDUALS_ASSOCIATION();
 665         } else if (unit.isInstanceOf(FieldObservation.class)
 666                 || unit.isInstanceOf(Observation.class)) {
 667             return Feature.OBSERVATION();
 668         } else if (unit.isInstanceOf(Fossil.class)
 669                 || unit.isInstanceOf(LivingBeing.class)
 670                 || unit.isInstanceOf(Specimen.class)) {
 671             return Feature.SPECIMEN();
 672         }
 673         if (DEBUG) {
 674             logger.warn("No feature defined for derived unit class: "
 675                     + unit.getClass().getSimpleName());
 676         }
 677         return null;
 678     }
 679
 680     private void makeIndividualsAssociation(Taxon taxon, DeterminationEvent determinationEvent) {
 681         //        try{
 682         //            taxon = (Taxon) getTaxonService().find(taxon.getUuid());
 683         //        }
 684         //        catch(Exception e){
 685         //            //logger.info("taxon uptodate");
 686         //        }
 687
 688         TaxonDescription taxonDescription=null;
 689         if (taxon.getTitleCache().contains("Chenopodium vulvaria L.")) {
 690             taxonDescription = (TaxonDescription) getDescriptionService().find(UUID.fromString("a60074e7-979b-41fd-ad4d-d391db474ac4"));
 691         }
 692         if(!taxon.getTitleCache().contains("Chenopodium vulvaria L.") || taxonDescription ==null) {
 693             taxonDescription = getTaxonDescription(taxon, ref, false, true);
 694             taxonDescription.setTitleCache(ref.getTitleCache(), true);
 695         }
 696
 697
 698         taxon.addDescription(taxonDescription);
 699
 700         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
 701         Feature feature = makeFeature(derivedUnitBase);
 702         indAssociation.setAssociatedSpecimenOrObservation(derivedUnitBase);
 703         indAssociation.setFeature(feature);
 704
 705         for (Reference<?> citation : determinationEvent.getReferences()) {
 706             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.PrimaryTaxonomicSource, null, null, citation, null));
 707         }
 708
 709         taxonDescription.addElement(indAssociation);
 710         taxonDescription.setTaxon(taxon);
 711
 712         getDescriptionService().saveOrUpdate(taxonDescription);
 713         getTaxonService().saveOrUpdate(taxon);
 714     }
 715
 716
 717     /* (non-Javadoc)
 718      * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IoStateBase)
 719      */
 720     @Override
 721     protected boolean isIgnore(SpecimenSynthesysExcelImportState state) {
 722         return false;
 723     }
 724
 725
 726     /* (non-Javadoc)
 727      * @see eu.etaxonomy.cdm.io.specimen.SpecimenIoBase#doInvoke(eu.etaxonomy.cdm.io.specimen.abcd206.SpecimenImportState)
 728      */
 729     @Override
 730     protected void doInvoke(SpecimenSynthesysExcelImportState state) {
 731         boolean success = true;
 732         if (state.getConfig().doAskForDate()) {
 733             keepAtomisedDate = askQuestion("Gathering dates can be stored in either atomised fieds (day month year) or in a concatenated field."+
 734                     "\nWhich value do you want to store?\nPress 1 for the atomised, press 2 for the concatenated field, and then press enter.");
 735         }
 736
 737         tx = startTransaction();
 738
 739         ref = state.getConfig().getTaxonReference();
 740         if (ref == null){
 741             ref = state.getConfig().getSourceReference();
 742         }
 743         getReferenceService().saveOrUpdate(ref);
 744         setClassification(state);
 745
 746         refreshTransaction();
 747
 748         URI source = state.getConfig().getSource();
 749         ArrayList<HashMap<String,String>> unitsList = null;
 750         try{
 751             unitsList = ExcelUtils.parseXLS(source);
 752         } catch(FileNotFoundException e){
 753             String message = "File not found: " + source;
 754             warnProgress(state, message, e);
 755             logger.error(message);
 756         }
 757
 758         logger.info("unitslist : "+unitsList.size());
 759         if (unitsList != null){
 760             //load collectors in the database
 761             prepareCollectors(unitsList,state);
 762             HashMap<String,String> unit=null;
 763             MyHashMap<String,String> myunit;
 764             for (int i=0; i<unitsList.size();i++){
 765                 if (i%100==0) {
 766                     logger.info("NbUnit prepared: "+i);
 767                     refreshTransaction();
 768                 }
 769                 unit = unitsList.get(i);
 770                 myunit=new MyHashMap<String, String>();
 771                 for (String key :unit.keySet()) {
 772                     myunit.put(key, unit.get(key));
 773                 }
 774                 //FIXME do this via state
 775                 setUnitPropertiesExcel(myunit, state.getConfig().getDefaultAuthor());//and then invoke
 776                 success &= start(state.getConfig());
 777             }
 778         }
 779         if (success == false){
 780             state.setUnsuccessfull();
 781         }
 782         commitTransaction(tx);
 783         return;
 784     }
 785
 786
 787     /**
 788      * @param unitsList
 789      * @param state
 790      */
 791     private void prepareCollectors(ArrayList<HashMap<String, String>> unitsList, SpecimenSynthesysExcelImportState state) {
 792         List<String> collectors = new ArrayList<String>();
 793         List<String> teams = new ArrayList<String>();
 794         List<List<String>> collectorinteams = new ArrayList<List<String>>();
 795         String tmp;
 796         for (HashMap<String,String> unit:unitsList){
 797             tmp=null;
 798             tmp = unit.get("collector");
 799             if (tmp != null && !tmp.isEmpty()) {
 800                 if (tmp.indexOf("et al.") != -1 || tmp.indexOf(" al. ")!=-1 || tmp.indexOf("& al.")!=-1 ) {
 801                     if (!tmp.trim().isEmpty()) {
 802                         teams.add(tmp.trim());
 803                     }
 804                 } else{
 805                     if(tmp.indexOf(" et ")!=-1 || tmp.indexOf("&")!=-1){
 806                         List<String> collteam = new ArrayList<String>();
 807                         String[] tmp1 = tmp.split(" et ");
 808                         for (String elt:tmp1){
 809                             String tmp2[] = elt.split("&");
 810                             for (String elt2:tmp2) {
 811                                 if (!elt2.trim().isEmpty()) {
 812                                     collectors.add(elt2.trim());
 813                                     collteam.add(elt2.trim());
 814                                 }
 815                             }
 816                         }
 817                         if (collteam.size()>0) {
 818                             collectorinteams.add(new ArrayList<String>(new HashSet<String>(collteam)));
 819                         }
 820                     }
 821                     else
 822                         if (!tmp.trim().isEmpty() && !tmp.toString().trim().equalsIgnoreCase("none")) {
 823                             collectors.add(tmp.trim());
 824                         }
 825                 }
 826             }
 827         }
 828
 829         List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
 830         List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
 831
 832
 833         //existing teams in DB
 834         Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
 835         List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
 836
 837         Set<UUID> uuids = new HashSet<UUID>();
 838         for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
 839             uuids.add(hibernateT.getUuid());
 840         }
 841         if (!uuids.isEmpty()){
 842             List<AgentBase> existingTeams = getAgentService().find(uuids);
 843             for (AgentBase existingP:existingTeams){
 844                 titleCacheTeam.put(existingP.getTitleCache(),(Team) existingP);
 845             }
 846         }
 847
 848
 849         Map<String,UUID> teamMap = new HashMap<String, UUID>();
 850         for (UuidAndTitleCache<Team> uuidt:hiberTeam){
 851             teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
 852         }
 853
 854         //existing persons in DB
 855         List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
 856         Map<String,Person> titleCachePerson = new HashMap<String, Person>();
 857         uuids = new HashSet<UUID>();
 858         for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
 859             uuids.add(hibernateP.getUuid());
 860         }
 861
 862         if (!uuids.isEmpty()){
 863             List<AgentBase> existingPersons = getAgentService().find(uuids);
 864             for (AgentBase existingP:existingPersons){
 865                 titleCachePerson.put(existingP.getTitleCache(),(Person) existingP);
 866             }
 867         }
 868
 869         Map<String,UUID> personMap = new HashMap<String, UUID>();
 870         for (UuidAndTitleCache<Person> person:hiberPersons){
 871             personMap.put(person.getTitleCache(), person.getUuid());
 872         }
 873
 874         java.util.Collection<AgentBase> personToadd = new ArrayList<AgentBase>();
 875         java.util.Collection<AgentBase> teamToAdd = new ArrayList<AgentBase>();
 876
 877         for (String collector:collectorsU){
 878             Person p = Person.NewInstance();
 879             p.setTitleCache(collector,true);
 880             if (!personMap.containsKey(p.getTitleCache())){
 881                 personToadd.add(p);
 882             }
 883         }
 884         for (String team:teamsU){
 885             Team p = Team.NewInstance();
 886             p.setTitleCache(team,true);
 887             if (!teamMap.containsKey(p.getTitleCache())){
 888                 teamToAdd.add(p);
 889             }
 890         }
 891         Map<UUID, AgentBase> uuuidPerson = getAgentService().save(personToadd);
 892         for (UUID u:uuuidPerson.keySet()){
 893             titleCachePerson.put(uuuidPerson.get(u).getTitleCache(),(Person) uuuidPerson.get(u) );
 894         }
 895
 896
 897         Person ptmp ;
 898         Map <String,Integer>teamdone = new HashMap<String, Integer>();
 899         for (List<String> collteam: collectorinteams){
 900             if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
 901                 Team team = new Team();
 902                 boolean em =true;
 903                 for (String collector:collteam){
 904                     ptmp = Person.NewInstance();
 905                     ptmp.setTitleCache(collector,true);
 906                     Person p2 = titleCachePerson.get(ptmp.getTitleCache());
 907                     team.addTeamMember(p2);
 908                     em=false;
 909                 }
 910                 if (!em) {
 911                     teamToAdd.add(team);
 912                 }
 913                 teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
 914             }
 915         }
 916
 917
 918         Map<UUID, AgentBase> uuuidTeam =  getAgentService().save(teamToAdd);
 919
 920         for (UUID u:uuuidTeam.keySet()){
 921             titleCacheTeam.put(uuuidTeam.get(u).getTitleCache(), (Team) uuuidTeam.get(u) );
 922         }
 923
 924         state.getConfig().setTeams(titleCacheTeam);
 925         state.getConfig().setPersons(titleCachePerson);
 926     }
 927
 928     private boolean askQuestion(String question){
 929         Scanner scan = new Scanner(System.in);
 930         System.out.println(question);
 931         int index = scan.nextInt();
 932         if (index == 1) {
 933             return true;
 934         } else {
 935             return false;
 936         }
 937     }
 938
 939
 940
 941
 942     /* (non-Javadoc)
 943      * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 944      */
 945     @Override
 946     protected boolean doCheck(SpecimenSynthesysExcelImportState state) {
 947         logger.warn("Validation not yet implemented for " + getClass().getSimpleName());
 948         return true;
 949     }
 950
 951
 952 }