app-import/src/main/java/eu/etaxonomy/cdm/io/bogota/BogotaSpecimenImport.java

   1 // $Id$
   2 /**
   3 * Copyright (C) 2017 EDIT
   4 * European Distributed Institute of Taxonomy
   5 * http://www.e-taxonomy.eu
   6 *
   7 * The contents of this file are subject to the Mozilla Public License Version 1.1
   8 * See LICENSE.TXT at the top of this package for the full license terms.
   9 */
  10 package eu.etaxonomy.cdm.io.bogota;
  11
  12 import java.util.Arrays;
  13 import java.util.HashMap;
  14 import java.util.List;
  15 import java.util.Map;
  16 import java.util.UUID;
  17
  18 import org.apache.log4j.Logger;
  19 import org.joda.time.Partial;
  20 import org.springframework.stereotype.Component;
  21
  22 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  23 import eu.etaxonomy.cdm.common.CdmUtils;
  24 import eu.etaxonomy.cdm.io.common.utils.ImportDeduplicationHelper;
  25 import eu.etaxonomy.cdm.model.agent.AgentBase;
  26 import eu.etaxonomy.cdm.model.agent.Institution;
  27 import eu.etaxonomy.cdm.model.agent.Person;
  28 import eu.etaxonomy.cdm.model.agent.Team;
  29 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  30 import eu.etaxonomy.cdm.model.common.CdmBase;
  31 import eu.etaxonomy.cdm.model.common.ExtensionType;
  32 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  33 import eu.etaxonomy.cdm.model.common.Language;
  34 import eu.etaxonomy.cdm.model.common.TimePeriod;
  35 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  36 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  37 import eu.etaxonomy.cdm.model.location.Country;
  38 import eu.etaxonomy.cdm.model.location.NamedArea;
  39 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  40 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  41 import eu.etaxonomy.cdm.model.location.Point;
  42 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  43 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  44 import eu.etaxonomy.cdm.model.name.Rank;
  45 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  46 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  47 import eu.etaxonomy.cdm.model.name.TaxonName;
  48 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  49 import eu.etaxonomy.cdm.model.occurrence.Collection;
  50 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  51 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  52 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  53 import eu.etaxonomy.cdm.model.reference.Reference;
  54 import eu.etaxonomy.cdm.model.taxon.Synonym;
  55 import eu.etaxonomy.cdm.model.taxon.Taxon;
  56 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  57 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  58 import eu.etaxonomy.cdm.model.term.DefinedTerm;
  59 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  60 import eu.etaxonomy.cdm.strategy.parser.DeterminationModifierParser;
  61 import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
  62 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  63 import eu.etaxonomy.cdm.strategy.parser.SpecimenTypeParser;
  64 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  65
  66 /**
  67  * @author a.mueller
  68  * @since 21.04.2017
  69  *
  70  */
  71 @Component
  72 public class BogotaSpecimenImport<CONFIG extends BogotaSpecimenImportConfigurator>
  73         extends SimpleExcelSpecimenImport<CONFIG> {
  74
  75     private static final long serialVersionUID = -884838817884874228L;
  76     @SuppressWarnings("unused")
  77     private static final Logger logger = Logger.getLogger(BogotaSpecimenImport.class);
  78
  79     private static final String COL_TAXON_UUID = "Platform Name ID = cdmID";
  80     private static final String COL_VOUCHER_ID = "Voucher ID";
  81
  82     private static final String COL_FAMILY = "Family";
  83     private static final String COL_GENUS = "Genus";
  84     private static final String COL_SPECIFIC_EPI = "Specific Epithet";
  85     private static final String COL_BASIONYM_AUTHOR = "Author in parenthesis";
  86     private static final String COL_AUTHOR = "Author";
  87     private static final String COL_IDENTIFIER = "Identifier";
  88     private static final String COL_IDENTIFICATION_DATE = "Identification date";
  89     private static final String COL_IDENTIFICATION_QUALIFIER = "Qualifier";
  90     private static final String COL_TYPUS = "Type";
  91     private static final String COL_IDENTIFICATION_HISTORY = "Identification history";
  92     private static final String COL_COLLECTOR_VERBATIM = "Verbatim Collectors  (Originalfeld JBB)";
  93     private static final String COL_COLLECTOR_LASTNAME = "Primary Collector Last Name  (Originalfeld JBB)";
  94     private static final String COL_COLLECTOR_FIRSTNAME = "Primary Collector First Name Initial (Originalfeld JBB)";
  95     private static final String COL_COLLECTOR_MIDDLENAME = "Primary Collector Middle Name Initial (Originalfeld JBB)";
  96     private static final String COL_COLLECTOR_TYPE = "Primary collector type (Originalfeld JBB)";
  97     private static final String COL_COLLECTOR_NUMBER = "Collector's No";
  98     private static final String COL_COLLECTORS = "Collectors";
  99     private static final String COL_COLLECTION_DATE_FROM = "Collection Date from";
 100     private static final String COL_COLLECTION_DATE_TO = "Collection Date to";
 101     private static final String COL_ALTITUDE_FROM = "Altitude Value from";
 102     private static final String COL_ALTITUDE_TO = "Altitude Value to";
 103     private static final String COL_ALTITUDE_UNIT = "Altitude Unit";
 104     private static final String COL_LOCALITY = "Locality";
 105     private static final String COL_LOCALITY_ID = "LocalityID";
 106     private static final String COL_LATITUDE = "Latitude";
 107     private static final String COL_LONGITUDE = "Longitude";
 108     private static final String COL_ERROR_DISTANCE = "Error distance in m";
 109     private static final String COL_COUNTRY = "Country";
 110     private static final String COL_STATE_AREA = "State/Province/Greater Area";
 111     private static final String COL_GEO_METHOD = "Geocode Method";
 112     private static final String COL_HABITUS = "Habitus";
 113     private static final String COL_COLLECTION = "[Series] Voucher location";
 114
 115     private static final UUID uuidAnonymous = UUID.fromString("2303f043-6e92-4afa-9082-7719e78a1c8a");
 116     private static final UUID uuidBogota = UUID.fromString("95b6cb03-8452-4439-98bd-8c1aa3c1da4e");
 117     private static final UUID uuidDefaultGeocodMethod = UUID.fromString("0983e680-b0ca-4e46-8df7-0f1d757a2e01");
 118     private static final UUID uuidExtTypeIdentificationHistory = UUID.fromString("7cee5c29-e16b-4e6f-ad57-bf7044259375");
 119     private static final UUID uuidDetQualVelAff = UUID.fromString("511a0c23-2646-4035-b570-36bdc2eb5557");
 120
 121 //    @SuppressWarnings("unchecked")
 122     private ImportDeduplicationHelper<SimpleExcelSpecimenImportState<?>> deduplicationHelper;
 123
 124     private final Map<String, TaxonNode> taxonNodeMap = new HashMap<>();
 125     private Reference secRef;
 126
 127     @Override
 128     protected String getWorksheetName(CONFIG config) {
 129         return "To be imported";
 130     }
 131
 132     /**
 133      * {@inheritDoc}
 134      */
 135     @Override
 136     protected void firstPass(SimpleExcelSpecimenImportState<CONFIG> state) {
 137
 138         Map<String, String> record = state.getOriginalRecord();
 139
 140         String voucherId = getValue(record, COL_VOUCHER_ID);
 141         if (!isInInterval(state)){
 142             return;
 143         }
 144         String line = state.getCurrentLine() + " (id:"+ voucherId+"): ";
 145         if (state.getCurrentLine() % 100 == 0){System.out.println(line);}
 146         try {
 147
 148             //species
 149             TaxonBase<?> taxonBase = getTaxonByCdmId(state, line, record, voucherId);
 150             if (taxonBase != null){
 151                 handleRecordForTaxon(state, voucherId, line, taxonBase);
 152             }else if (record.get(COL_TAXON_UUID)!= null){
 153                 //  do nothing
 154             }else{
 155                 taxonBase = getOrCreateNewTaxon(state, record, line);
 156                 handleRecordForTaxon(state, voucherId, line, taxonBase);
 157             }
 158
 159         } catch (Exception e) {
 160             state.getResult().addError("An unexpected exception appeared in record", e, null, line);
 161             e.printStackTrace();
 162         }
 163
 164     }
 165
 166     /**
 167      * @param state
 168      * @param record
 169      * @param line
 170      * @return
 171      */
 172     private Taxon getOrCreateNewTaxon(SimpleExcelSpecimenImportState<CONFIG> state,
 173             Map<String, String> record, String line) {
 174         String familyStr = record.get(COL_FAMILY);
 175         String genusStr = record.get(COL_GENUS);
 176         initTaxonMap(state);
 177         TaxonName speciesName = makeSpeciesName(state, line);
 178         String titleCache = speciesName.getTitleCache();
 179         TaxonNode existingSpeciesNode = taxonNodeMap.get(titleCache);
 180         if (existingSpeciesNode != null){
 181             return existingSpeciesNode.getTaxon();
 182         }else{
 183             Reference sec = getSecReference(state);
 184             Taxon newTaxon = Taxon.NewInstance(speciesName, sec);
 185             newTaxon.addSource(makeOriginalSource(state));
 186             TaxonNode existingGenusNode = taxonNodeMap.get(genusStr);
 187             if (existingGenusNode == null){
 188                 TaxonName genusName = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
 189                 genusName.setGenusOrUninomial(genusStr);
 190                 Taxon newGenus = Taxon.NewInstance(genusName, sec);
 191                 newGenus.addSource(makeOriginalSource(state));
 192                 TaxonNode existingFamilyNode = taxonNodeMap.get(familyStr);
 193                 if (existingFamilyNode == null){
 194                     TaxonName familyName = TaxonNameFactory.NewBotanicalInstance(Rank.FAMILY());
 195                     familyName.setGenusOrUninomial(familyStr);
 196                     Taxon newFamily = Taxon.NewInstance(familyName, sec);
 197                     newFamily.addSource(makeOriginalSource(state));
 198                     TaxonNode plantaeNode = taxonNodeMap.get("Plantae");
 199                     existingFamilyNode = plantaeNode.addChildTaxon(newFamily, null, null);
 200                     save(existingFamilyNode);
 201                 }
 202                 existingGenusNode = existingFamilyNode.addChildTaxon(newGenus, null, null);
 203                 save(existingGenusNode);
 204             }
 205             existingSpeciesNode = existingGenusNode.addChildTaxon(newTaxon, null, null);
 206             save(existingSpeciesNode);
 207             return newTaxon;
 208         }
 209
 210     }
 211
 212     /**
 213      * @param existingFamilyNode
 214      */
 215     private void save(TaxonNode node) {
 216         getTaxonNodeService().saveOrUpdate(node);
 217         taxonNodeMap.put(node.getTaxon().getName().getTitleCache(), node);
 218
 219     }
 220
 221     /**
 222      * @param state
 223      * @return
 224      */
 225     private Reference getSecReference(SimpleExcelSpecimenImportState<CONFIG> state) {
 226         if (this.secRef == null){
 227             Reference sec = state.getConfig().getSecReference();
 228             this.secRef = getReferenceService().find(sec.getUuid());
 229             if (this.secRef == null){
 230                 this.secRef = sec;
 231                 getReferenceService().save(sec);
 232             }
 233         }
 234
 235
 236         return this.secRef;
 237     }
 238
 239     /**
 240      * @param state
 241      * @param record
 242      * @param line
 243      * @return
 244      */
 245     private TaxonName makeSpeciesName(SimpleExcelSpecimenImportState<CONFIG> state, String line) {
 246         Map<String, String> record = state.getOriginalRecord();
 247         String genus = record.get(COL_GENUS);
 248         String species = record.get(COL_SPECIFIC_EPI);
 249         String basionymAuthorStr = record.get(COL_BASIONYM_AUTHOR);
 250         String authorStr = record.get(COL_AUTHOR);
 251         INonViralNameParser<?> parser = NonViralNameParserImpl.NewInstance();
 252         String fullName = genus + " " +  species +
 253                 (basionymAuthorStr == null ? "" : " ("+basionymAuthorStr+")")
 254                 + " " + authorStr;
 255         TaxonName newName = (TaxonName)parser.parseFullName(fullName , NomenclaturalCode.ICNAFP, Rank.SPECIES());
 256         String titleCache = newName.getTitleCache();
 257         if (newName.isProtectedTitleCache()){
 258             state.getResult().addWarning("Name not parsable: " +  fullName);
 259         }
 260         if (taxonNodeMap.get(titleCache)== null){
 261             getDeduplicationHelper(state).replaceAuthorNamesAndNomRef(state, newName);
 262             newName.addSource(makeOriginalSource(state));
 263         }
 264
 265         return newName;
 266     }
 267
 268     /**
 269      * @param state
 270      *
 271      */
 272     private void initTaxonMap(SimpleExcelSpecimenImportState<CONFIG> state) {
 273         if (taxonNodeMap.isEmpty()){
 274             List<String> propertyPaths = Arrays.asList(new String[]{"taxon.name"});
 275             List<TaxonNode> list = getTaxonNodeService().list(null, null, null, null, propertyPaths);
 276             for (TaxonNode node : list){
 277                 if (node.getTaxon()!= null){
 278                     String strName = node.getTaxon().getName().getTitleCache();
 279                     TaxonNode existingNode = taxonNodeMap.get(strName);
 280                     if (existingNode != null){
 281                         state.getResult().addWarning("Taxon name exists more than once while initializing taxon map: " + strName, "initTaxonMap");
 282                     }else{
 283                         taxonNodeMap.put(strName, node);
 284                     }
 285                 }
 286             }
 287         }
 288     }
 289
 290     /**
 291      * @param state
 292      * @param record
 293      * @param voucherId
 294      * @param line
 295      * @param taxonBase
 296      * @param taxon
 297      */
 298     protected void handleRecordForTaxon(SimpleExcelSpecimenImportState<CONFIG> state,
 299             String voucherId, String line, TaxonBase<?> taxonBase) {
 300
 301         Map<String, String> record = state.getOriginalRecord();
 302         Taxon taxon = getTaxon(taxonBase);
 303
 304         TaxonDescription taxonDescription = getTaxonDescription(state, line, taxon);
 305
 306         DerivedUnit specimen = makeSpecimen(state, line, record, voucherId, taxonBase);
 307
 308         IndividualsAssociation indAssoc = IndividualsAssociation.NewInstance(specimen);
 309         indAssoc.addImportSource(voucherId, COL_VOUCHER_ID, getSourceCitation(state), null);
 310         taxonDescription.addElement(indAssoc);
 311     }
 312
 313
 314     /**
 315      * @param state
 316      * @return
 317      */
 318     private boolean isInInterval(SimpleExcelSpecimenImportState<CONFIG> state) {
 319         Integer min = state.getConfig().getMinLineNumber();
 320         Integer max = state.getConfig().getMaxLineNumber();
 321         Integer current = state.getCurrentLine();
 322         if (current < min || current > max){
 323             return false;
 324         }else{
 325             return true;
 326         }
 327     }
 328
 329
 330     /**
 331      * @param state
 332      * @param line
 333      * @param taxon
 334      * @return
 335      */
 336     private TaxonDescription getTaxonDescription(SimpleExcelSpecimenImportState<CONFIG> state, String line,
 337             Taxon taxon) {
 338         Reference ref = getSourceCitation(state);
 339         TaxonDescription desc = this.getTaxonDescription(taxon, ref, ! IMAGE_GALLERY, ! CREATE);
 340         if (desc == null){
 341             //TODO move title creation into base method
 342             desc = this.getTaxonDescription(taxon, ref, ! IMAGE_GALLERY, CREATE);
 343             desc.setTitleCache("Specimen Excel import for " +  taxon.getName().getTitleCache(), true);
 344         }
 345         return desc;
 346     }
 347
 348
 349     /**
 350      * @param state
 351      * @param line
 352      * @param record
 353      * @param voucherId
 354      * @return
 355      */
 356     private DerivedUnit makeSpecimen(SimpleExcelSpecimenImportState<CONFIG> state, String line,
 357             Map<String, String> record, String voucherId, TaxonBase<?> taxonBase) {
 358
 359         DerivedUnitFacade facade = DerivedUnitFacade.NewPreservedSpecimenInstance();
 360         facade.setAccessionNumber(voucherId);
 361         makeDetermination(facade.innerDerivedUnit(), state, line, record, taxonBase.getName());
 362         makeTypus(facade.innerDerivedUnit(), state, line, record, taxonBase.getName());
 363         makeCollectorFields(facade, state, line, record);
 364         makeLocationFields(facade, state, line, record);
 365         makeHabitus(facade, state, line, record);
 366         makeCollection(facade, state, line, record);
 367         DerivedUnit specimen = facade.innerDerivedUnit();
 368         specimen.addSource(makeOriginalSource(state));
 369         FieldUnit fieldUnit = facade.innerFieldUnit();
 370         fieldUnit.addSource(makeOriginalSource(state));
 371         return specimen;
 372     }
 373
 374
 375     /**
 376      * @param innerDerivedUnit
 377      * @param state
 378      * @param line
 379      * @param record
 380      * @param name
 381      */
 382     private void makeTypus(DerivedUnit specimen, SimpleExcelSpecimenImportState<CONFIG> state, String line,
 383             Map<String, String> record, TaxonName name) {
 384         String typus = record.get(COL_TYPUS);
 385         if (typus != null){
 386             SpecimenTypeDesignationStatus status;
 387             try {
 388                 status = SpecimenTypeParser.parseSpecimenTypeStatus(typus);
 389                 SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
 390                 designation.setTypeStatus(status);
 391                 name.addSpecimenTypeDesignation(specimen, status, null, null, null, false, false);
 392             } catch (UnknownCdmTypeException e) {
 393                 state.getResult().addWarning("Type designation could not be parsed", null, line);
 394             }
 395         }
 396     }
 397
 398
 399     /**
 400      * @param facade
 401      * @param state
 402      * @param line
 403      * @param record
 404      */
 405     private void makeCollection(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state,
 406             String line, Map<String, String> record) {
 407         String strCollection = record.get(COL_COLLECTION);
 408         String collectionFormat = ".*\\([A-Z]{2,4}\\)";
 409         if (!strCollection.matches(collectionFormat)){
 410             String message = "Voucher location format does not match the expected format. Voucher '(" + strCollection + ")' location not added.";
 411             state.getResult().addError(message, null, line);
 412             return;
 413         }
 414         String[] splits = strCollection.split("\\(");
 415         String collectionName = splits[0];
 416         String collectionCode = splits[1].replace(")", "");
 417         Collection collection = Collection.NewInstance();
 418         collection.setName(collectionName);
 419         collection.setCode(collectionCode);
 420         collection = getDeduplicationHelper(state).getExistingCollection(state, collection);
 421         facade.setCollection(collection);
 422     }
 423
 424
 425     /**
 426      * @param state
 427      * @return
 428      */
 429     private ImportDeduplicationHelper<SimpleExcelSpecimenImportState<?>> getDeduplicationHelper(SimpleExcelSpecimenImportState<CONFIG> state) {
 430         if (deduplicationHelper == null){
 431             deduplicationHelper = ImportDeduplicationHelper.NewInstance(this, state);
 432         }
 433         return deduplicationHelper;
 434     }
 435
 436
 437     /**
 438      * @param facade
 439      * @param state
 440      * @param line
 441      * @param record
 442      */
 443     private void makeHabitus(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
 444             Map<String, String> record) {
 445         String habitus = record.get(COL_HABITUS);
 446         if (habitus != null){
 447             facade.setPlantDescription(habitus);
 448         }
 449     }
 450
 451
 452     /**
 453      * @param facade
 454      * @param state
 455      * @param line
 456      * @param record
 457      * @param voucherId
 458      */
 459     private void makeLocationFields(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
 460             Map<String, String> record) {
 461         //Altitude
 462         String strAltitudeFrom = record.get(COL_ALTITUDE_FROM);
 463         String strAltitudeTo = record.get(COL_ALTITUDE_TO);
 464         Integer intAltitudeFrom = intFromString(state, strAltitudeFrom, line, COL_ALTITUDE_FROM);
 465         Integer intAltitudeTo = intFromString(state, strAltitudeTo, line, COL_ALTITUDE_TO);
 466         if (intAltitudeFrom != null){
 467             facade.setAbsoluteElevation(intAltitudeFrom);
 468             if (!intAltitudeFrom.equals(intAltitudeTo)){
 469                 facade.setAbsoluteElevationMax(intAltitudeTo);
 470             }
 471             if (!record.get(COL_ALTITUDE_UNIT).equals("m")){
 472                 state.getResult().addWarning("Altitude unit is not m but " + record.get(COL_ALTITUDE_UNIT), "makeLocationFields", line);
 473             }
 474         }
 475         checkNoToIfNoFrom(strAltitudeFrom, strAltitudeTo, state, line, COL_ALTITUDE_TO);
 476
 477         //locality
 478         String locality = record.get(COL_LOCALITY);
 479         if (locality != null){  //should always exist
 480             facade.setLocality(locality, Language.SPANISH_CASTILIAN());
 481         }
 482
 483         //Lat + Long
 484         String strLatitude = record.get(COL_LATITUDE);
 485         String strLongitude = record.get(COL_LONGITUDE);
 486         String strError = record.get(COL_ERROR_DISTANCE);
 487         Double dblLatitude = doubleFromString(state, strLatitude, line, COL_LATITUDE);
 488         Double dblLongitude = doubleFromString(state, strLongitude, line, COL_LONGITUDE);
 489         Integer intError = intFromString(state, strError, line, COL_ERROR_DISTANCE);
 490         ReferenceSystem referenceSystem = makeReferenceSystem(state, record, line);
 491
 492         if (dblLatitude != null || dblLongitude != null || intError != null){ //should always exist
 493             Point exactLocation = Point.NewInstance(dblLongitude, dblLatitude, referenceSystem, intError);
 494             facade.setExactLocation(exactLocation);
 495         }
 496
 497         //Country
 498         String strCountry = record.get(COL_COUNTRY);
 499         if (strCountry != null){
 500             if (strCountry.equals("Colombia")){
 501                 Country colombia = Country.COLOMBIAREPUBLICOF();
 502                 colombia.setLabel("Colombia");
 503                 getTermService().saveOrUpdate(colombia);
 504                 facade.setCountry(colombia);
 505             }else{
 506                 state.getResult().addWarning("Country was not Colombia as expected but " +  strCountry,
 507                         "makeLocationFields", line);
 508             }
 509         }
 510
 511         //State
 512         String strStateArea = record.get(COL_STATE_AREA);
 513         if (strStateArea != null){
 514             if (strStateArea.replaceAll("\\s", "").equalsIgnoreCase("Bogotá,D.C.")){
 515                 NamedArea bogota = makeBogota(state, line);
 516                 facade.addCollectingArea(bogota);
 517             }else{
 518                 state.getResult().addWarning(COL_STATE_AREA + " was not 'Bogotá,  D.C.' as expected but " +  strCountry,
 519                         "makeLocationFields", line);
 520             }
 521         }
 522     }
 523
 524
 525     /**
 526      * @param strAltitudeFrom
 527      * @param strAltitudeTo
 528      * @param state
 529      * @param line
 530      * @param colAltitudeTo
 531      */
 532     private void checkNoToIfNoFrom(String strFrom, String strTo,
 533             SimpleExcelSpecimenImportState<CONFIG> state,
 534             String line, String toAttributeName) {
 535         if (isNotBlank(strTo) && isBlank(strFrom)){
 536             String message = "A min-max attribute has a max value (%s) but no min value. This is invalid."
 537                     + " The max value attribute name is %s.";
 538             message = String.format(message, strTo, toAttributeName);
 539             state.getResult().addWarning(message, null, line);
 540         }
 541     }
 542
 543     private ReferenceSystem defaultGeocodeMethod;
 544
 545     /**
 546      * @param state
 547      * @param record
 548      * @param line
 549      * @return
 550      */
 551     private ReferenceSystem makeReferenceSystem(SimpleExcelSpecimenImportState<CONFIG> state,
 552             Map<String, String> record, String line) {
 553         String defaultStrRefSys = "Wieczorek, J., Guo, Q., & Hijmans, R. (2004). The point-radius method for georeferencing locality descriptions and calculating associated uncertainty. International journal of geographical information science, 18(8), 745-767.; Escobar D, Díaz SR, Jojoa LM, Rudas E, Albarracín RD, Ramírez C, Gómez JY, López CR, Saavedra J (2015). Georreferenciación de localidades: Una guía de referencia para colecciones biológicas. Instituto de Investigación de Recursos Biológicos Alexander von Humboldt – Instituto de Ciencias Naturales, Universidad Nacional de Colombia. Bogotá D.C., Colombia. 95 p.";
 554         String strRefSys = record.get(COL_GEO_METHOD);
 555         if (strRefSys == null){
 556             return null;
 557         }else if (!strRefSys.equals(defaultStrRefSys)){
 558             state.getResult().addError("The expected Geocode Method is not the expected default method. Geocode Method was not added.", null, line);
 559             return null;
 560         }else if (defaultGeocodeMethod != null){
 561             return defaultGeocodeMethod;
 562         }else{
 563             String label = "Point radius method";
 564             String description = defaultStrRefSys;
 565             String labelAbbrev = "PRM";
 566             defaultGeocodeMethod = getReferenceSystem(state, uuidDefaultGeocodMethod,
 567                     label, description, labelAbbrev, null);
 568             return defaultGeocodeMethod;
 569         }
 570     }
 571
 572     private NamedArea bogota;
 573     /**
 574      * @param state
 575      * @param line
 576      * @return
 577      */
 578     private NamedArea makeBogota(SimpleExcelSpecimenImportState<CONFIG> state, String line) {
 579         if (bogota != null){
 580             return bogota;
 581         }else{
 582             String label = "Bogotá, D.C.";
 583             NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
 584             NamedAreaLevel level = NamedAreaLevel.STATE();
 585             bogota = getNamedArea(state, uuidBogota, label, label, null, areaType,
 586                     level, null, null, null);
 587             return bogota;
 588         }
 589     }
 590
 591
 592     /**
 593      * @param facade
 594      * @param state
 595      * @param line
 596      * @param record
 597      */
 598     private void makeCollectorFields(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
 599             Map<String, String> record) {
 600
 601         //collector number
 602         facade.setFieldNumber(record.get(COL_COLLECTOR_NUMBER));
 603
 604         //gathering date
 605         String dateFrom = unknownToNull((record.get(COL_COLLECTION_DATE_FROM)));
 606         String dateTo = unknownToNull(record.get(COL_COLLECTION_DATE_TO));
 607         checkNoToIfNoFrom(dateFrom, dateTo, state, line, COL_COLLECTION_DATE_TO);
 608         try {
 609             if (dateFrom != null && dateFrom.equals(dateTo)){
 610                 dateTo = null;
 611             }
 612             TimePeriod gatheringPeriod = TimePeriodParser.parseEnglishDate(dateFrom, dateTo);
 613             facade.setGatheringPeriod(gatheringPeriod);
 614         } catch (Exception e) {
 615             state.getResult().addError("Error creating gathering date", e, null, line);
 616         }
 617
 618         //collector
 619         String collectorType = record.get(COL_COLLECTOR_TYPE);
 620         String collectors = record.get(COL_COLLECTORS);
 621         AgentBase<?> collector;
 622         if (collectorType.startsWith("Anonymous")){
 623             collector = getAnonymous();
 624         }else if (collectorType.equals("Brother") || collectorType.equals("Person")){
 625             Person person = Person.NewInstance();
 626             if (collectorType.equals("Person")){
 627                 person.setFamilyName(record.get(COL_COLLECTOR_LASTNAME));
 628                 String initials = CdmUtils.concat("", record.get(COL_COLLECTOR_FIRSTNAME), record.get(COL_COLLECTOR_MIDDLENAME));
 629                 initials = (initials == null)? null : initials.replaceAll("\\s", "");
 630                 person.setInitials(initials);
 631                 String full = person.getTitleCache();
 632                 if (!full.equals(collectors)){
 633                     person.setTitleCache(collectors, true);
 634                     //TODO use setCollectorTitle in future
 635                 }
 636             }else{
 637                 person.setTitleCache(collectors, true);
 638                 person.setPrefix("Hno.");
 639                 person.setGivenName(collectors.replace("Hno.", "").trim());
 640             }
 641             collector = person;
 642         }else if (collectorType.equals("Group")){
 643             collector = Team.NewTitledInstance(collectors, collectors);
 644         }else if (collectorType.equals("Institution")){
 645             collector = Institution.NewNamedInstance(collectors);
 646         }else{
 647             String message = "Collector type " + collectorType + " not yet supported by import. Collector not added.";
 648             state.getResult().addError(message, null, line);
 649             collector = null;
 650         }
 651         collector = getDeduplicationHelper(state).getExistingAgent(state, collector);
 652         facade.setCollector(collector);
 653     }
 654
 655
 656     /**
 657      * @param string
 658      * @return
 659      */
 660     private String unknownToNull(String string) {
 661         if (string == null || string.equalsIgnoreCase("unknown")){
 662             return null;
 663         }else{
 664             return string;
 665         }
 666     }
 667
 668     private Person anonymous;
 669     private Person getAnonymous() {
 670         if (anonymous != null){
 671             return anonymous;
 672         }
 673         anonymous = CdmBase.deproxy(getAgentService().find(uuidAnonymous), Person.class);
 674         if (anonymous == null){
 675             anonymous = Person.NewTitledInstance("Anon.");
 676             anonymous.setUuid(uuidAnonymous);
 677             getAgentService().save(anonymous);
 678         }
 679         return anonymous;
 680     }
 681
 682
 683     /**
 684      * @param facade
 685      * @param state
 686      * @param line
 687      * @param record
 688      * @param taxonBase
 689      */
 690     private void makeDetermination(DerivedUnit specimen, SimpleExcelSpecimenImportState<CONFIG> state, String line,
 691             Map<String, String> record, TaxonName taxonName) {
 692
 693         DeterminationEvent determination;
 694         determination = DeterminationEvent.NewInstance(taxonName, specimen);
 695         determination.setPreferredFlag(true);
 696
 697         //determiner/identifier
 698         TeamOrPersonBase<?> determiner = makeDeterminer(state, record, line);
 699         determination.setDeterminer(determiner);
 700
 701         //date
 702         TimePeriod date = makeIdentificationDate(state, record, line);
 703         determination.setTimeperiod(date);
 704
 705         //qualifier
 706         DefinedTerm qualifier = makeDeterminationQualifier(state, record, line);
 707         determination.setModifier(qualifier);
 708
 709         //history
 710         String history = record.get(COL_IDENTIFICATION_HISTORY);
 711         if (history != null){
 712             String label = "Identification History";
 713             String text = label;
 714             ExtensionType detHistoryType = getExtensionType(state, uuidExtTypeIdentificationHistory, label, text, null);
 715             specimen.addExtension(history, detHistoryType);
 716         }
 717     }
 718
 719
 720     /**
 721      * @param state
 722      * @param record
 723      * @param line
 724      * @return
 725      */
 726     private TeamOrPersonBase<?> makeDeterminer(SimpleExcelSpecimenImportState<CONFIG> state,
 727             Map<String, String> record, String line) {
 728         String identifier = record.get(COL_IDENTIFIER);
 729         if (identifier == null){
 730             return null;
 731         }else if (identifier.equals("Anon.")){
 732             return getAnonymous();
 733         }else{
 734             Person person = Person.NewInstance();
 735             person.setTitleCache(identifier, true);
 736
 737             String[] splits = identifier.split("\\.");
 738             int length = splits.length;
 739             if (splits[length - 1].equals("")){
 740                 splits[length - 2]= splits[length - 2]+".";
 741                 length--;
 742             }
 743             if (splits[length - 1].startsWith("-")){
 744                 splits[length - 2]= splits[length - 2]+"." + splits[length - 1];
 745                 length--;
 746             }
 747             String familyName = splits[length - 1];
 748             String initials = null;
 749             for (int i= 0; i < length-1;i++){
 750                 initials = CdmUtils.concat("", initials, splits[i]+".");
 751             }
 752             person.setFamilyName(familyName);
 753             person.setInitials(initials);
 754             TeamOrPersonBase<?> result = getDeduplicationHelper(state).getExistingAuthor(state, person);
 755             return result;
 756         }
 757     }
 758
 759
 760     /**
 761      * @param state
 762      * @param record
 763      * @param line
 764      * @return
 765      */
 766     private TimePeriod makeIdentificationDate(SimpleExcelSpecimenImportState<CONFIG> state,
 767             Map<String, String> record, String line) {
 768         String strDate = record.get(COL_IDENTIFICATION_DATE);
 769         if (strDate == null || strDate.equals("s.n.")){
 770             return null;
 771         }
 772         String[] splits = strDate.split("/");
 773         String strYear = splits[splits.length-1];
 774         String strMonth = splits.length < 2? null:splits[splits.length-2];
 775         String strDay = splits.length < 3? null:splits[splits.length-3];
 776
 777         Integer year = intFromString(state, strYear, line, COL_IDENTIFICATION_DATE);
 778         Integer month = intFromString(state, strMonth, line, COL_IDENTIFICATION_DATE);
 779         Integer day = intFromString(state, strDay, line, COL_IDENTIFICATION_DATE);
 780         Partial start = TimePeriodParser.makePartialFromDateParts(year, month, day);
 781         return TimePeriod.NewInstance(start);
 782     }
 783
 784
 785     /**
 786      * @param state
 787      * @param record
 788      * @param line
 789      * @return
 790      */
 791     private DefinedTerm makeDeterminationQualifier(SimpleExcelSpecimenImportState<CONFIG> state,
 792             Map<String, String> record, String line) {
 793         String qualifier = record.get(COL_IDENTIFICATION_QUALIFIER);
 794         if (qualifier != null){
 795             try {
 796                 return DeterminationModifierParser.parseDeterminationQualifier(qualifier);
 797             } catch (UnknownCdmTypeException e) {
 798                 //TODO add to terms
 799                 if (qualifier.equals("vel. aff.")){
 800
 801                     DefinedTerm velAff = (DefinedTerm)getTermService().find(uuidDetQualVelAff);
 802                     if (velAff == null){
 803                         velAff = DefinedTerm.NewModifierInstance(qualifier, qualifier, qualifier);
 804                         velAff.setUuid(uuidDetQualVelAff);
 805                         getTermService().save(velAff);
 806                     }
 807                     return velAff;
 808                 }
 809                 state.getResult().addError("Determination qualifier could not be recognized: " + qualifier, null, line);
 810                 return null;
 811             }
 812         }else{
 813             return null;
 814         }
 815     }
 816
 817
 818     /**
 819      * @param taxonBase
 820      * @return
 821      */
 822     private Taxon getTaxon(TaxonBase<?> taxonBase) {
 823         if (taxonBase.isInstanceOf(Synonym.class)){
 824             return CdmBase.deproxy(taxonBase, Synonym.class).getAcceptedTaxon();
 825         }else{
 826             return CdmBase.deproxy(taxonBase, Taxon.class);
 827         }
 828     }
 829
 830
 831     /**
 832      * @param state
 833      * @param line
 834      * @param record
 835      * @param noStr
 836      * @return
 837      */
 838     private TaxonBase<?> getTaxonByCdmId(SimpleExcelSpecimenImportState<CONFIG> state, String line,
 839             Map<String, String> record, String noStr) {
 840
 841         String strUuidTaxon = record.get(COL_TAXON_UUID);
 842         if (strUuidTaxon != null && ! state.getConfig().isOnlyNonCdmTaxa()){
 843             UUID uuidTaxon;
 844             try {
 845                 uuidTaxon = UUID.fromString(strUuidTaxon);
 846             } catch (Exception e) {
 847                 state.getResult().addError("Taxon uuid has incorrect format. Taxon could not be loaded. Data not imported.", null, line);
 848                 return null;
 849             }
 850             TaxonBase<?> result = getTaxonService().find(uuidTaxon);
 851             if (result == null){
 852                 state.getResult().addError("Taxon for uuid  "+strUuidTaxon+" could not be found in database. "
 853                         + "Taxon could not be loaded. Data not imported.", null, line);
 854
 855             }
 856             return result;
 857         }else{
 858             return null;
 859         }
 860     }
 861
 862     @Override
 863     protected IdentifiableSource makeOriginalSource(SimpleExcelSpecimenImportState<CONFIG> state) {
 864         return IdentifiableSource.NewDataImportInstance(getValue(state.getOriginalRecord(), COL_VOUCHER_ID), COL_VOUCHER_ID, getSourceCitation(state));
 865     }
 866
 867     /**
 868      * @param state
 869      * @return
 870      */
 871     protected Reference getSourceCitation(SimpleExcelSpecimenImportState<CONFIG> state) {
 872         Reference source = state.getConfig().getSourceReference();
 873         if (source.getId() == 0){
 874             Reference persisted = getReferenceService().find(source.getUuid());
 875             if (persisted == null){
 876                 getReferenceService().saveOrUpdate(source);
 877             }else{
 878                 state.getConfig().setSourceReference(persisted);
 879                 source = persisted;
 880             }
 881         }
 882         return source;
 883     }
 884 }