cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenCdmExcelImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen.excel.in;
  11
  12 import java.text.ParseException;
  13 import java.util.ArrayList;
  14 import java.util.List;
  15 import java.util.UUID;
  16
  17 import org.apache.commons.lang.StringUtils;
  18 import org.apache.log4j.Logger;
  19 import org.springframework.stereotype.Component;
  20
  21 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  22 import eu.etaxonomy.cdm.api.service.config.MatchingTaxonConfigurator;
  23 import eu.etaxonomy.cdm.common.CdmUtils;
  24 import eu.etaxonomy.cdm.io.common.ICdmIO;
  25 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  26 import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase.PostfixTerm;
  27 import eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase;
  28 import eu.etaxonomy.cdm.io.specimen.excel.in.SpecimenRow.DeterminationLight;
  29 import eu.etaxonomy.cdm.model.agent.AgentBase;
  30 import eu.etaxonomy.cdm.model.agent.Person;
  31 import eu.etaxonomy.cdm.model.agent.Team;
  32 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  33 import eu.etaxonomy.cdm.model.common.Annotation;
  34 import eu.etaxonomy.cdm.model.common.AnnotationType;
  35 import eu.etaxonomy.cdm.model.common.CdmBase;
  36 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  37 import eu.etaxonomy.cdm.model.common.Language;
  38 import eu.etaxonomy.cdm.model.common.TimePeriod;
  39 import eu.etaxonomy.cdm.model.description.Feature;
  40 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  41 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  42 import eu.etaxonomy.cdm.model.location.NamedArea;
  43 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  44 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  45 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  46 import eu.etaxonomy.cdm.model.location.Country;
  47 import eu.etaxonomy.cdm.model.name.BotanicalName;
  48 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  49 import eu.etaxonomy.cdm.model.name.NonViralName;
  50 import eu.etaxonomy.cdm.model.name.Rank;
  51 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  52 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  53 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  54 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  55 import eu.etaxonomy.cdm.model.occurrence.Collection;
  56 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  57 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  58 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  59 import eu.etaxonomy.cdm.model.reference.Reference;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  61 import eu.etaxonomy.cdm.model.taxon.Taxon;
  62 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  63 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  64 import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
  65 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  66 import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
  67 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  68 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  69
  70 /**
  71  * @author a.mueller
  72  * @created 10.05.2011
  73  * @version 1.0
  74  */
  75 @Component
  76 public class SpecimenCdmExcelImport  extends ExcelTaxonOrSpecimenImportBase<SpecimenCdmExcelImportState, SpecimenRow>  implements ICdmIO<SpecimenCdmExcelImportState> {
  77         private static final Logger logger = Logger.getLogger(SpecimenCdmExcelImport.class);
  78
  79         private static final String WORKSHEET_NAME = "Specimen";
  80
  81         private static final String BASIS_OF_RECORD_COLUMN = "(?i)(BasisOfRecord)";
  82         private static final String COUNTRY_COLUMN = "(?i)(Country)";
  83         private static final String AREA_COLUMN = "(?i)(Area)";
  84         private static final String ISO_COUNTRY_COLUMN = "(?i)(ISOCountry|CountryCode)";
  85         private static final String LOCALITY_COLUMN = "(?i)(Locality)";
  86         private static final String ALTITUDE_COLUMN = "(?i)(AbsoluteElevation|Altitude)";
  87         private static final String ALTITUDE_MAX_COLUMN = "(?i)(AbsoluteElevation|Altitude)Max(imum)?";
  88         private static final String COLLECTION_DATE_COLUMN = "(?i)(CollectionDate)";
  89         private static final String COLLECTION_DATE_END_COLUMN = "(?i)(CollectionDateEnd)";
  90         private static final String COLLECTOR_COLUMN = "(?i)(Collector)";
  91         private static final String COLLECTORS_COLUMN = "(?i)(Collectors)";
  92         private static final String PRIMARY_COLLECTOR_COLUMN = "(?i)(PrimaryCollector)";
  93         private static final String LONGITUDE_COLUMN = "(?i)(Longitude)";
  94         private static final String LATITUDE_COLUMN = "(?i)(Latitude)";
  95         private static final String REFERENCE_SYSTEM_COLUMN = "(?i)(ReferenceSystem)";
  96         private static final String ERROR_RADIUS_COLUMN = "(?i)(ErrorRadius)";
  97
  98
  99         private static final String COLLECTORS_NUMBER_COLUMN = "(?i)((Collectors|Field)Number)";
 100         private static final String ECOLOGY_COLUMN = "(?i)(Ecology|Habitat)";
 101         private static final String PLANT_DESCRIPTION_COLUMN = "(?i)(PlantDescription)";
 102         private static final String FIELD_NOTES_COLUMN = "(?i)(FieldNotes)";
 103         private static final String SEX_COLUMN = "(?i)(Sex)";
 104
 105
 106         private static final String ACCESSION_NUMBER_COLUMN = "(?i)(AccessionNumber)";
 107         private static final String BARCODE_COLUMN = "(?i)(Barcode)";
 108         private static final String COLLECTION_CODE_COLUMN = "(?i)(CollectionCode)";
 109         private static final String COLLECTION_COLUMN = "(?i)(Collection)";
 110         private static final String UNIT_NOTES_COLUMN = "(?i)((Unit)?Notes)";
 111
 112
 113         private static final String TYPE_CATEGORY_COLUMN = "(?i)(TypeCategory)";
 114         private static final String TYPIFIED_NAME_COLUMN = "(?i)(TypifiedName|TypeOf)";
 115
 116
 117         private static final String SOURCE_COLUMN = "(?i)(Source)";
 118         private static final String ID_IN_SOURCE_COLUMN = "(?i)(IdInSource)";
 119
 120
 121         private static final String DETERMINATION_AUTHOR_COLUMN = "(?i)(Author)";
 122         private static final String DETERMINATION_MODIFIER_COLUMN = "(?i)(DeterminationModifier)";
 123         private static final String DETERMINED_BY_COLUMN = "(?i)(DeterminationBy)";
 124         private static final String DETERMINED_WHEN_COLUMN = "(?i)(Det(ermination)?When)";
 125         private static final String DETERMINATION_NOTES_COLUMN = "(?i)(DeterminationNote)";
 126         private static final String EXTENSION_COLUMN = "(?i)(Ext(ension)?)";
 127
 128
 129         public SpecimenCdmExcelImport() {
 130                 super();
 131         }
 132
 133
 134
 135
 136         @Override
 137         protected void analyzeSingleValue(KeyValue keyValue, SpecimenCdmExcelImportState state) {
 138                 SpecimenRow row = state.getCurrentRow();
 139                 String value = keyValue.value;
 140                 if(keyValue.key.matches(BASIS_OF_RECORD_COLUMN)) {
 141                         row.setBasisOfRecord(value);
 142                 } else if(keyValue.key.matches(COUNTRY_COLUMN)) {
 143                         row.setCountry(value);
 144                 } else if(keyValue.key.matches(ISO_COUNTRY_COLUMN)) {
 145                         row.setIsoCountry(value);
 146                 } else if(keyValue.key.matches(LOCALITY_COLUMN)) {
 147                         row.setLocality(value);
 148                 } else if(keyValue.key.matches(FIELD_NOTES_COLUMN)) {
 149                         row.setLocality(value);
 150                 } else if(keyValue.key.matches(ALTITUDE_COLUMN)) {
 151                         row.setAltitude(value);
 152                 } else if(keyValue.key.matches(ALTITUDE_MAX_COLUMN)) {
 153                         row.setAltitudeMax(value);
 154                 } else if(keyValue.key.matches(COLLECTOR_COLUMN)) {
 155                         row.putCollector(keyValue.index, value);
 156                 } else if(keyValue.key.matches(PRIMARY_COLLECTOR_COLUMN)) {
 157                         row.setPrimaryCollector(value);
 158                 } else if(keyValue.key.matches(ECOLOGY_COLUMN)) {
 159                         row.setEcology(value);
 160                 } else if(keyValue.key.matches(PLANT_DESCRIPTION_COLUMN)) {
 161                         row.setPlantDescription(value);
 162                 } else if(keyValue.key.matches(SEX_COLUMN)) {
 163                         row.setSex(value);
 164                 } else if(keyValue.key.matches(COLLECTION_DATE_COLUMN)) {
 165                         row.setCollectingDate(value);
 166                 } else if(keyValue.key.matches(COLLECTION_DATE_END_COLUMN)) {
 167                         row.setCollectingDateEnd(value);
 168                 } else if(keyValue.key.matches(COLLECTORS_COLUMN)) {
 169                         row.setCollectors(value);
 170                 } else if(keyValue.key.matches(COLLECTOR_COLUMN)) {
 171                         row.putCollector(keyValue.index, value);
 172                 } else if(keyValue.key.matches(COLLECTORS_NUMBER_COLUMN)) {
 173                         row.setCollectorsNumber(value);
 174                 } else if(keyValue.key.matches(LONGITUDE_COLUMN)) {
 175                         row.setLongitude(value);
 176                 } else if(keyValue.key.matches(LATITUDE_COLUMN)) {
 177                         row.setLatitude(value);
 178                 } else if(keyValue.key.matches(REFERENCE_SYSTEM_COLUMN)) {
 179                         row.setReferenceSystem(value);
 180                 } else if(keyValue.key.matches(ERROR_RADIUS_COLUMN)) {
 181                         row.setErrorRadius(value);
 182                 } else if(keyValue.key.matches(AREA_COLUMN)) {
 183                         if (keyValue.postfix != null){
 184                                 row.addLeveledArea(keyValue.postfix, value);
 185                         }else{
 186                                 logger.warn("Not yet implemented");
 187                         }
 188                 } else if(keyValue.key.matches(LANGUAGE)) {
 189                         row.setLanguage(value);
 190
 191
 192                 } else if(keyValue.key.matches(ACCESSION_NUMBER_COLUMN)) {
 193                         row.setAccessionNumber(value);
 194                 } else if(keyValue.key.matches(BARCODE_COLUMN)) {
 195                         row.setBarcode(value);
 196                 } else if(keyValue.key.matches(UNIT_NOTES_COLUMN)) {
 197                         row.putUnitNote(keyValue.index, value);
 198
 199
 200                 } else if(keyValue.key.matches(FAMILY_COLUMN)) {
 201                         row.putDeterminationFamily(keyValue.index, value);
 202                 } else if(keyValue.key.matches(GENUS_COLUMN)) {
 203                         row.putDeterminationGenus(keyValue.index, value);
 204                 } else if(keyValue.key.matches(SPECIFIC_EPITHET_COLUMN)) {
 205                         row.putDeterminationSpeciesEpi(keyValue.index, value);
 206                 } else if(keyValue.key.matches(INFRASPECIFIC_EPITHET_COLUMN)) {
 207                         row.putDeterminationInfraSpeciesEpi(keyValue.index, value);
 208                 } else if(keyValue.key.matches(RANK_COLUMN)) {
 209                         row.putDeterminationRank(keyValue.index, value);
 210                 } else if(keyValue.key.matches(TAXON_UUID_COLUMN)) {
 211                         row.putDeterminationTaxonUuid(keyValue.index, value);
 212                 } else if(keyValue.key.matches(FULL_NAME_COLUMN)) {
 213                         row.putDeterminationFullName(keyValue.index, value);
 214                 } else if(keyValue.key.matches(DETERMINATION_AUTHOR_COLUMN)) {
 215                         row.putDeterminationAuthor(keyValue.index, value);
 216                 } else if(keyValue.key.matches(DETERMINATION_MODIFIER_COLUMN)) {
 217                         row.putDeterminationDeterminationModifier(keyValue.index, value);
 218                 } else if(keyValue.key.matches(DETERMINATION_NOTES_COLUMN)) {
 219                         row.putDeterminationDeterminationNotes(keyValue.index, value);
 220                 } else if(keyValue.key.matches(DETERMINED_BY_COLUMN)) {
 221                         row.putDeterminationDeterminedBy(keyValue.index, value);
 222                 } else if(keyValue.key.matches(DETERMINED_WHEN_COLUMN)) {
 223                         row.putDeterminationDeterminedWhen(keyValue.index, value);
 224
 225                 } else if(keyValue.key.matches(COLLECTION_CODE_COLUMN)) {
 226                         row.setCollectionCode(value);
 227                 } else if(keyValue.key.matches(COLLECTION_COLUMN)) {
 228                         row.setCollection(value);
 229
 230                 } else if(keyValue.key.matches(TYPE_CATEGORY_COLUMN)) {
 231                         row.putTypeCategory(keyValue.index, getSpecimenTypeStatus(state, value));
 232                 } else if(keyValue.key.matches(TYPIFIED_NAME_COLUMN)) {
 233                         row.putTypifiedName(keyValue.index, getTaxonName(state, value));
 234
 235
 236                 } else if(keyValue.key.matches(SOURCE_COLUMN)) {
 237                         row.putSourceReference(keyValue.index, getOrMakeReference(state, value) );
 238                 } else if(keyValue.key.matches(ID_IN_SOURCE_COLUMN)) {
 239                         row.putIdInSource(keyValue.index, value);
 240                 } else if(keyValue.key.matches(EXTENSION_COLUMN)) {
 241                         if (keyValue.postfix != null){
 242                                 row.addExtension(keyValue.postfix, value);
 243                         }else{
 244                                 logger.warn("Extension without postfix not yet implemented");
 245                         }
 246
 247                 }else {
 248                         state.setUnsuccessfull();
 249                         logger.error("Unexpected column header " + keyValue.originalKey);
 250                 }
 251
 252         return;
 253         }
 254
 255
 256         @Override
 257         protected void firstPass(SpecimenCdmExcelImportState state) {
 258                 SpecimenRow row = state.getCurrentRow();
 259
 260                 //basis of record
 261                 SpecimenOrObservationType type = SpecimenOrObservationType.valueOf2(row.getBasisOfRecord());
 262                 if (type == null){
 263                         String message = "%s is not a valid BasisOfRecord. 'Unknown' is used instead in line %d.";
 264                         message = String.format(message, row.getBasisOfRecord(), state.getCurrentLine());
 265                         logger.warn(message);
 266                         type = SpecimenOrObservationType.DerivedUnit;
 267                 }
 268                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(type);
 269
 270
 271                 Language lang = Language.DEFAULT();
 272                 if (StringUtils.isNotBlank(row.getLanguage())){
 273                         Language langIso = getTermService().getLanguageByIso(row.getLanguage());
 274                         if (langIso == null){
 275                                 String message = "Language could not be recognized: %s. Use default language instead. Line %d.";
 276                                 message = String.format(message, langIso, state.getCurrentLine());
 277                         }else{
 278                                 lang = langIso;
 279                         }
 280                 }
 281
 282                 //country
 283                 handleCountry(facade, row, state);
 284                 handleAreas(facade,row, state);
 285
 286                 facade.setGatheringPeriod(getTimePeriod(row.getCollectingDate(), row.getCollectingDateEnd()));
 287                 facade.setLocality(row.getLocality());
 288                 facade.setFieldNotes(row.getFieldNotes());
 289                 facade.setFieldNumber(row.getCollectorsNumber());
 290                 facade.setEcology(row.getEcology(), lang);
 291                 facade.setPlantDescription(row.getPlantDescription(), lang);
 292 //              facade.setSex(row.get)
 293                 handleExactLocation(facade, row, state);
 294                 facade.setCollector(getOrMakeAgent(state, row.getCollectors()));
 295                 facade.setPrimaryCollector(getOrMakePrimaryCollector(facade, row.getPrimaryCollector(), state));
 296                 handleAbsoluteElevation(facade, row, state);
 297
 298                 //derivedUnit
 299                 facade.setBarcode(row.getBarcode());
 300                 facade.setAccessionNumber(row.getAccessionNumber());
 301                 facade.setCollection(getOrMakeCollection(state, row.getCollectionCode(), row.getCollection()));
 302                 for (IdentifiableSource source : row.getSources()){
 303                         facade.addSource(source);
 304                 }
 305                 for (SpecimenTypeDesignation designation : row.getTypeDesignations()){
 306                         logger.warn("FIXME"); //FIXME
 307 //                      facade.innerDerivedUnit().addSpecimenTypeDesignation(designation);
 308                 }
 309                 handleDeterminations(state, row, facade);
 310                 handleExtensions(facade.innerDerivedUnit(),row, state);
 311                 for (String note : row.getUnitNotes()){
 312                         Annotation annotation = Annotation.NewInstance(note, AnnotationType.EDITORIAL(), Language.DEFAULT());
 313                         facade.addAnnotation(annotation);
 314                 }
 315
 316                 //save
 317                 getOccurrenceService().save(facade.innerDerivedUnit());
 318                 return;
 319         }
 320
 321         private void handleAbsoluteElevation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 322                 //altitude
 323
 324                 try {
 325                         String altitude = row.getAltitude();
 326                         if (StringUtils.isBlank(altitude)){
 327                                 return;
 328                         }
 329 //                      if (altitude.endsWith(".0")){
 330 //                              altitude = altitude.substring(0, altitude.length() -2);
 331 //                      }
 332                         int value = Integer.valueOf(altitude);
 333                         facade.setAbsoluteElevation(value);
 334                 } catch (NumberFormatException e) {
 335                         String message = "Absolute elevation / altitude '%s' is not an integer number in line %d";
 336                         message = String.format(message, row.getAltitude(), state.getCurrentLine());
 337                         logger.warn(message);
 338                         return;
 339                 }
 340
 341                 //max
 342
 343                 try {
 344                         String max = row.getAltitudeMax();
 345                         if (StringUtils.isBlank(max)){
 346                                 return;
 347                         }
 348 //                      if (max.endsWith(".0")){
 349 //                              max = max.substring(0, max.length() -2);
 350 //                      }
 351                         int value = Integer.valueOf(max);
 352                         //TODO avoid unequal distance
 353                         int min = facade.getAbsoluteElevation();
 354                         if ( (value - min) % 2 == 1 ){
 355                                 String message = "Altitude min-max difference ist not equal. Max reduced by 1 in line %d";
 356                                 message = String.format(message, state.getCurrentLine());
 357                                 logger.warn(message);
 358                                 value--;
 359                         }
 360                         facade.setAbsoluteElevationRange(min, value);
 361                 } catch (NumberFormatException e) {
 362                         String message = "Absolute elevation / Altitude maximum '%s' is not an integer number in line %d";
 363                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 364                         logger.warn(message);
 365                         return;
 366                 }catch (Exception e){
 367                         String message = "Error occurred when trying to write Absolute elevation / Altitude maximum '%s' in line %d";
 368                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 369                         logger.warn(message);
 370                         return;
 371
 372                 }
 373
 374
 375         }
 376
 377         private void handleAreas(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 378                 List<PostfixTerm> areas = row.getLeveledAreas();
 379
 380                 for (PostfixTerm lArea : areas){
 381                         String description = lArea.term;
 382                         String abbrev = lArea.term;
 383                         NamedAreaType type = null;
 384                         String key = lArea.postfix + "_" + lArea.term;
 385                         UUID areaUuid = state.getArea(key);
 386                         NamedAreaLevel level = state.getPostfixLevel(lArea.postfix);
 387
 388                         TermMatchMode matchMode = state.getConfig().getAreaMatchMode();
 389                         NamedArea area = getNamedArea(state, areaUuid, lArea.term, description, abbrev, type, level, null, matchMode);
 390                         facade.addCollectingArea(area);
 391                         if (areaUuid == null){
 392                                 state.putArea(key, area.getUuid());
 393                         }
 394                 }
 395         }
 396
 397
 398         /**
 399          * @param state
 400          * @param row
 401          * @param facade
 402          */
 403         private void handleDeterminations(SpecimenCdmExcelImportState state,SpecimenRow row, DerivedUnitFacade facade) {
 404                 boolean isFirstDetermination = true;
 405                 DeterminationLight commonDetermination = row.getCommonDetermination();
 406                 Taxon commonTaxon = null;
 407                 TaxonNameBase<?,?> commonName = null;
 408
 409                 boolean hasCommonTaxonInfo = (commonDetermination == null) ? false : commonDetermination.hasTaxonInformation();
 410                 if (hasCommonTaxonInfo && commonDetermination != null){
 411                         TaxonBase<?> taxonBase = null;
 412                         if (StringUtils.isNotBlank(commonDetermination.taxonUuid)){
 413                                 UUID taxonUuid = UUID.fromString(commonDetermination.taxonUuid);
 414                                 taxonBase = getTaxonService().find(taxonUuid);
 415                                 if (taxonBase == null){
 416                                         String message = "Taxon for uuid %s not found in line %d.";
 417                                         message = String.format(message, taxonUuid.toString(), state.getCurrentLine());
 418                                         logger.warn(message);
 419                                 }
 420                         }else{
 421                                 taxonBase = findBestMatchingTaxon(state, commonDetermination, state.getConfig().isCreateTaxonIfNotExists());
 422                         }
 423                         commonTaxon = getAcceptedTaxon(taxonBase);
 424                         if (taxonBase != null){
 425                                 commonName = taxonBase.getName();
 426                         }else{
 427                                 commonTaxon = createTaxonFromDetermination(state, commonDetermination);
 428                                 commonName = commonTaxon.getName();
 429                         }
 430                 }
 431
 432
 433                 for (DeterminationLight determinationLight : row.getDetermination()){
 434                         Taxon taxon;
 435                         if (! hasCommonTaxonInfo){
 436                                 taxon = findBestMatchingTaxon(state, determinationLight, state.getConfig().isCreateTaxonIfNotExists());
 437                         }else{
 438                                 taxon = commonTaxon;
 439                         }
 440                         if (taxon != null){
 441                                 getTaxonService().saveOrUpdate(taxon);
 442                                 if (state.getConfig().isMakeIndividualAssociations() && taxon != null){
 443                                         IndividualsAssociation indivAssociciation = IndividualsAssociation.NewInstance();
 444                                         DerivedUnit du = facade.innerDerivedUnit();
 445                                         indivAssociciation.setAssociatedSpecimenOrObservation(du);
 446                                         getTaxonDescription(taxon).addElement(indivAssociciation);
 447                                         Feature feature = Feature.INDIVIDUALS_ASSOCIATION();
 448                                         if (facade.getType().isPreservedSpecimen()){
 449                                                 feature = Feature.SPECIMEN();
 450                                         }else if (facade.getType().isFeatureObservation()){
 451                                                 feature = Feature.OBSERVATION();
 452                                         }
 453                                         if (state.getConfig().isUseMaterialsExaminedForIndividualsAssociations()){
 454                                                 feature = Feature.MATERIALS_EXAMINED();
 455                                         }
 456
 457                                         indivAssociciation.setFeature(feature);
 458                                 }
 459                                 if (state.getConfig().isDeterminationsAreDeterminationEvent()){
 460                                         DeterminationEvent detEvent = makeDeterminationEvent(state, determinationLight, taxon);
 461                                         detEvent.setPreferredFlag(isFirstDetermination);
 462                                         facade.addDetermination(detEvent);
 463                                 }
 464                         }
 465
 466                         if (isFirstDetermination && state.getConfig().isFirstDeterminationIsStoredUnder()){
 467                                 TaxonNameBase<?,?> name;
 468
 469                                 if (!hasCommonTaxonInfo){
 470                                         name = findBestMatchingName(state, determinationLight);
 471                                 }else{
 472                                         if (commonName == null){
 473                                                 commonName = findBestMatchingName(state, commonDetermination);
 474                                         }
 475                                         name = commonName;
 476                                 }
 477                                 if (name != null){
 478                                         facade.setStoredUnder(name);
 479                                 }
 480                         }
 481                         isFirstDetermination = false;
 482                 }
 483         }
 484
 485         private Taxon createTaxonFromDetermination( SpecimenCdmExcelImportState state, DeterminationLight commonDetermination) {
 486
 487                 //rank
 488                 Rank rank;
 489                 try {
 490                         rank = StringUtils.isBlank(commonDetermination.rank) ? null : Rank.getRankByNameOrIdInVoc(commonDetermination.rank, true);
 491                 } catch (UnknownCdmTypeException e) {
 492                         rank = null;
 493                 }
 494
 495                 //name
 496                 NonViralName<?> name;
 497                 INonViralNameParser<NonViralName> parser = NonViralNameParserImpl.NewInstance();
 498                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 499                 if (StringUtils.isNotBlank(commonDetermination.fullName)){
 500                         name = parser.parseFullName(commonDetermination.fullName, nc, rank);
 501                         if (StringUtils.isBlank(name.getAuthorshipCache()) && StringUtils.isNotBlank(commonDetermination.author)){
 502                                 setAuthorship(name, commonDetermination.author, parser);
 503                         }
 504                 }else{
 505                         if (nc != null){
 506                                 name = (NonViralName)nc.getNewTaxonNameInstance(rank);
 507                         }else{
 508                                 name = NonViralName.NewInstance(rank);
 509                         }
 510                         if (StringUtils.isNotBlank(commonDetermination.genus)){
 511                                 name.setGenusOrUninomial(commonDetermination.genus);
 512                         }
 513                         if (StringUtils.isNotBlank(commonDetermination.speciesEpi)){
 514                                 name.setSpecificEpithet(commonDetermination.speciesEpi);
 515                         }
 516                         if (StringUtils.isNotBlank(commonDetermination.infraSpeciesEpi)){
 517                                 name.setInfraSpecificEpithet(commonDetermination.infraSpeciesEpi);
 518                         }
 519                         if (StringUtils.isNotBlank(commonDetermination.author)){
 520                                 setAuthorship(name, commonDetermination.author, parser);
 521                         }
 522                         //guess rank if null
 523                         if (name.getRank() == null){
 524                                 if (name.getInfraGenericEpithet() != null && name.getSpecificEpithet() == null){
 525                                         name.setRank(Rank.INFRAGENERICTAXON());
 526                                 }else if (name.getSpecificEpithet() != null && name.getInfraSpecificEpithet() == null){
 527                                         name.setRank(Rank.SPECIES());
 528                                 }else if (name.getInfraSpecificEpithet() != null){
 529                                         name.setRank(Rank.INFRASPECIFICTAXON());
 530                                 }
 531
 532                         }
 533
 534                 }
 535                 //sec
 536                 Reference<?> sec = null;
 537                 if (StringUtils.isNotBlank(commonDetermination.determinedBy)){
 538                         sec = ReferenceFactory.newGeneric();
 539                         TeamOrPersonBase<?> determinedBy;
 540                         BotanicalName dummyName = BotanicalName.NewInstance(Rank.SPECIES());
 541                         try {
 542                                 parser.parseAuthors(dummyName, commonDetermination.determinedBy);
 543                                 determinedBy = (TeamOrPersonBase<?>)dummyName.getCombinationAuthorTeam();
 544                         } catch (StringNotParsableException e) {
 545                                 determinedBy = Team.NewTitledInstance(commonDetermination.determinedBy, commonDetermination.determinedBy);
 546                         }
 547                         sec.setAuthorTeam(determinedBy);
 548                 }
 549
 550                 //taxon
 551                 Taxon taxon = Taxon.NewInstance(name, sec);
 552
 553                 if (StringUtils.isNotBlank(commonDetermination.family)){
 554                         if (name.getRank() == null || name.getRank().isLower(Rank.FAMILY()) ){
 555                                 logger.warn("Family taxon could not be created");
 556                         }
 557                 }
 558
 559                 //return
 560                 return taxon;
 561
 562         }
 563
 564
 565
 566
 567         private void setAuthorship(NonViralName<?> name, String author, INonViralNameParser<NonViralName> parser) {
 568                 if (name.isInstanceOf(BotanicalName.class) || name.isInstanceOf(ZoologicalName.class)){
 569                         try {
 570                                 parser.parseAuthors(name, author);
 571                         } catch (StringNotParsableException e) {
 572                                 name.setAuthorshipCache(author);
 573                         }
 574                 }else{
 575                         name.setAuthorshipCache(author);
 576                 }
 577
 578         }
 579
 580
 581
 582
 583         /**
 584          * This method tries to find the best matching taxon depending on the import configuration,
 585          * the taxon name information and the concept information available.
 586          *
 587          *
 588          * @param state
 589          * @param determinationLight
 590          * @param createIfNotExists
 591          * @return
 592          */
 593         private Taxon findBestMatchingTaxon(SpecimenCdmExcelImportState state, DeterminationLight determinationLight, boolean createIfNotExists) {
 594                 NonViralName<?> name = makeTaxonName(state, determinationLight);
 595
 596                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 597
 598                 if (! StringUtils.isBlank(titleCache)){
 599                         MatchingTaxonConfigurator matchConfigurator = MatchingTaxonConfigurator.NewInstance();
 600                         matchConfigurator.setTaxonNameTitle(titleCache);
 601                         matchConfigurator.setIncludeSynonyms(false);
 602                         Taxon taxon = getTaxonService().findBestMatchingTaxon(matchConfigurator);
 603
 604                         if(taxon == null && createIfNotExists){
 605                                 logger.info("creating new Taxon from TaxonName '" + titleCache+"'");
 606                                 UUID secUuid = null; //TODO
 607                                 Reference<?> sec = null;
 608                                 if (secUuid != null){
 609                                         sec = getReferenceService().find(secUuid);
 610                                 }
 611                                 taxon = Taxon.NewInstance(name, sec);
 612                         }else if (taxon == null){
 613                                 String message = "Taxon '%s' not found in line %d";
 614                                 message = String.format(message, titleCache, state.getCurrentLine());
 615                                 logger.warn(message);
 616                         }
 617                         return taxon;
 618                 }else {
 619                         return null;
 620                 }
 621         }
 622
 623         /**
 624          * @param state
 625          * @param determinationLight
 626          * @param name
 627          * @return
 628          */
 629         private String makeSearchNameTitleCache(SpecimenCdmExcelImportState state, DeterminationLight determinationLight,
 630                                 NonViralName<?> name) {
 631                 String titleCache = determinationLight.fullName;
 632                 if (! state.getConfig().isPreferNameCache() || StringUtils.isBlank(titleCache) ){
 633                         String computedTitleCache = name.getTitleCache();
 634                         if (StringUtils.isNotBlank(computedTitleCache)){
 635                                 titleCache = computedTitleCache;
 636                         }
 637
 638                 }
 639                 return titleCache;
 640         }
 641
 642         /**
 643          * @param state
 644          * @param determinationLight
 645          * @return
 646          */
 647         private NonViralName<?> makeTaxonName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 648                 NonViralName<?> name = NonViralName.NewInstance(null);
 649                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 650                 if (nc != null){
 651                         name = (NonViralName<?>)nc.getNewTaxonNameInstance(null);
 652                 }
 653                 name.setGenusOrUninomial(determinationLight.genus);
 654                 name.setSpecificEpithet(determinationLight.speciesEpi);
 655                 name.setInfraSpecificEpithet(determinationLight.infraSpeciesEpi);
 656
 657                 //FIXME bracketAuthors and teams not yet implemented!!!
 658                 List<String> authors = new ArrayList<String>();
 659                 if (StringUtils.isNotBlank(determinationLight.author)){
 660                         authors.add(determinationLight.author);
 661                 }
 662                 TeamOrPersonBase<?> agent = (TeamOrPersonBase)getOrMakeAgent(state, authors);
 663                 name.setCombinationAuthorTeam(agent);
 664
 665                 try {
 666                         if (StringUtils.isNotBlank(determinationLight.rank) ){
 667                                 name.setRank(Rank.getRankByNameOrIdInVoc(determinationLight.rank, nc, true));
 668                         }
 669                 } catch (UnknownCdmTypeException e) {
 670                         String message = "Rank not found: %s: ";
 671                         message = String.format(message, determinationLight.rank);
 672                         logger.warn(message);
 673                 }
 674                 if (StringUtils.isBlank(name.getInfraSpecificEpithet()) && StringUtils.isNotBlank(name.getSpecificEpithet() )){
 675                         name.setRank(Rank.SPECIES());
 676                 }
 677                 if (StringUtils.isBlank(name.getSpecificEpithet()) && StringUtils.isNotBlank(name.getGenusOrUninomial() )){
 678                         name.setRank(Rank.SPECIES());
 679                 }
 680                 if (StringUtils.isBlank(name.getTitleCache())){
 681                         //TODO test
 682                         name.setTitleCache(determinationLight.fullName, true);
 683                 }
 684                 return name;
 685         }
 686
 687         private TaxonNameBase findBestMatchingName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 688
 689                 NonViralName<?> name = makeTaxonName(state, determinationLight);
 690                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 691
 692                 //TODO
 693                 List<TaxonNameBase> matchingNames = getNameService().findByName(null, titleCache, MatchMode.EXACT, null, null, null, null, null).getRecords();
 694                 if (matchingNames.size() > 0){
 695                         return matchingNames.get(0);
 696                 } else if (matchingNames.size() > 0){
 697                         logger.warn("Get best matching taxon name not yet fully implemeted for specimen import");
 698                         return matchingNames.get(0);
 699                 }else{
 700                         return null;
 701                 }
 702
 703         }
 704
 705
 706         private DeterminationEvent makeDeterminationEvent(SpecimenCdmExcelImportState state, DeterminationLight determination, Taxon taxon) {
 707                 DeterminationEvent event = DeterminationEvent.NewInstance();
 708                 //taxon
 709                 event.setTaxon(taxon);
 710
 711                 //date
 712                 TimePeriod date = TimePeriodParser.parseString(determination.determinedWhen);
 713                 event.setTimeperiod(date);
 714                 //by
 715                 //FIXME bracketAuthors and teams not yet implemented!!!
 716                 List<String> authors = new ArrayList<String>();
 717                 if (StringUtils.isNotBlank(determination.determinedBy)){
 718                         authors.add(determination.determinedBy);
 719                 }
 720                 TeamOrPersonBase<?> actor = getOrMakeAgent(state, authors);
 721                 TeamOrPersonBase<?> secAuthor = taxon.getSec() == null ? null : taxon.getSec().getAuthorTeam();
 722                 if (actor != null && secAuthor != null && secAuthor.getTitleCache().equals(actor.getTitleCache()) && secAuthor.getNomenclaturalTitle().equals(actor.getNomenclaturalTitle())) {
 723                         actor = secAuthor;
 724                 }
 725
 726                 event.setActor(actor);
 727
 728                 //TODO
 729                 if (StringUtils.isNotBlank(determination.modifier)){
 730                         logger.warn("DeterminationModifiers not yet implemented for specimen import");
 731                 }
 732 //              DeterminationModifier modifier = DeterminationModifier.NewInstance(term, label, labelAbbrev);
 733 //              determination.modifier;
 734                 //notes
 735                 if (StringUtils.isNotEmpty(determination.notes)){
 736                         Annotation annotation = Annotation.NewInstance(determination.notes, AnnotationType.EDITORIAL(), Language.DEFAULT());
 737                         event.addAnnotation(annotation);
 738                 }
 739
 740                 return event;
 741         }
 742
 743         private TaxonDescription getTaxonDescription(Taxon taxon) {
 744                 TaxonDescription desc = this.getTaxonDescription(taxon, ! IMAGE_GALLERY, CREATE);
 745                 return desc;
 746         }
 747
 748         private TeamOrPersonBase<?> getOrMakeAgent(SpecimenCdmExcelImportState state, List<String> agents) {
 749                 if (agents.size() == 0){
 750                         return null;
 751                 }else if (agents.size() == 1){
 752                         return getOrMakePerson(state, agents.get(0));
 753                 }else{
 754                         return getOrMakeTeam(state, agents);
 755                 }
 756         }
 757
 758         private Person getOrMakePrimaryCollector(DerivedUnitFacade facade, String primaryCollector, SpecimenCdmExcelImportState state) {
 759                 if (StringUtils.isBlank(primaryCollector)){
 760                         return null;
 761                 }
 762                 AgentBase<?> collector = facade.getCollector();
 763                 List<Person> collectors = new ArrayList<Person>();
 764                 if (collector.isInstanceOf(Team.class) ){
 765                         Team team = CdmBase.deproxy(collector, Team.class);
 766                         collectors.addAll(team.getTeamMembers());
 767                 }else if (collector.isInstanceOf(Person.class)){
 768                         collectors.add(CdmBase.deproxy(collector, Person.class));
 769                 }else{
 770                         throw new IllegalStateException("Unknown subclass of agentbase: " + collector.getClass().getName() );
 771                 }
 772                 for (Person person :collectors){
 773                         if (primaryCollector.equalsIgnoreCase(person.getTitleCache())){
 774                                 return person;
 775                         }
 776                         if (primaryCollector.equalsIgnoreCase(person.getNomenclaturalTitle())){
 777                                 return person;
 778                         }
 779                 }
 780                 String message = "Primary Agent '%s' could not be determined in collector(s) in line %d";
 781                 message = String.format(message, primaryCollector, state.getCurrentLine());
 782                 logger.warn(message);
 783                 return null;
 784         }
 785
 786         private Team getOrMakeTeam(SpecimenCdmExcelImportState state, List<String> agents) {
 787                 String key = CdmUtils.concat("_", agents.toArray(new String[0]));
 788
 789                 Team result = state.getTeam(key);
 790                 if (result == null){
 791                         result = Team.NewInstance();
 792                         for (String member : agents){
 793                                 Person person = getOrMakePerson(state, member);
 794                                 result.addTeamMember(person);
 795                         }
 796                         state.putTeam(key, result);
 797                 }
 798                 return result;
 799         }
 800
 801         private Person getOrMakePerson(SpecimenCdmExcelImportState state, String value) {
 802                 Person result = state.getPerson(value);
 803                 if (result == null){
 804                         result = Person.NewInstance();
 805                         result.setTitleCache(value, true);
 806                         state.putPerson(value, result);
 807                 }
 808                 return result;
 809         }
 810
 811         private Reference<?> getOrMakeReference(SpecimenCdmExcelImportState state, String value) {
 812                 Reference<?> result = state.getReference(value);
 813                 if (result == null){
 814                         result = ReferenceFactory.newGeneric();
 815                         result.setTitleCache(value, true);
 816                         state.putReference(value, result);
 817                 }
 818                 return result;
 819         }
 820
 821
 822
 823         private Collection getOrMakeCollection(SpecimenCdmExcelImportState state, String collectionCode, String collectionString) {
 824                 Collection result = state.getCollection(collectionCode);
 825                 if (result == null){
 826                         result = Collection.NewInstance();
 827                         result.setCode(collectionCode);
 828                         result.setName(collectionString);
 829                         state.putCollection(collectionCode, result);
 830                 }
 831                 return result;
 832         }
 833
 834
 835         private TaxonNameBase<?, ?> getTaxonName(SpecimenCdmExcelImportState state, String name) {
 836                 TaxonNameBase<?,?> result = null;
 837                 result = state.getName(name);
 838                 if (result != null){
 839                         return result;
 840                 }
 841                 List<TaxonNameBase<?,?>> list = getNameService().findNamesByTitle(name);
 842                 //TODO better strategy to find best name, e.g. depending on the classification it is used in
 843                 if (! list.isEmpty()){
 844                         result = list.get(0);
 845                 }
 846                 if (result == null){
 847                         NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
 848                         NomenclaturalCode code = state.getConfig().getNomenclaturalCode();
 849                         result = parser.parseFullName(name, code, null);
 850
 851                 }
 852                 if (result != null){
 853                         state.putName(name, result);
 854                 }
 855                 return result;
 856         }
 857
 858         private SpecimenTypeDesignationStatus getSpecimenTypeStatus(SpecimenCdmExcelImportState state, String key)  {
 859                 SpecimenTypeDesignationStatus result = null;
 860                 try {
 861                         result = state.getTransformer().getSpecimenTypeDesignationStatusByKey(key);
 862                         if (result == null){
 863                                 String message = "Type status not recognized for %s in line %d";
 864                                 message = String.format(message, key, state.getCurrentLine());
 865                                 logger.warn(message);
 866                         }
 867                         return result;
 868                 } catch (UndefinedTransformerMethodException e) {
 869                         throw new RuntimeException("getSpecimenTypeDesignationStatusByKey not yet implemented");
 870                 }
 871
 872
 873         }
 874
 875
 876         private void handleExactLocation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 877
 878                 //reference system
 879                 ReferenceSystem refSys = null;
 880                 if (StringUtils.isNotBlank(row.getReferenceSystem())){
 881                         String strRefSys = row.getReferenceSystem().trim().replaceAll("\\s", "");
 882                         UUID refUuid;
 883                         try {
 884                                 refSys = state.getTransformer().getReferenceSystemByKey(strRefSys);
 885                                 if (refSys == null){
 886                                         //TODO we still need user defined Reference Systems here
 887                                         refUuid = state.getTransformer().getReferenceSystemUuid(strRefSys);
 888                                         if (refUuid == null){
 889                                                 String message = "Unknown reference system %s in line %d";
 890                                                 message = String.format(message, strRefSys, state.getCurrentLine());
 891                                                 logger.warn(message);
 892                                         }
 893                                         refSys = getReferenceSystem(state, refUuid, strRefSys, strRefSys, strRefSys, null);
 894                                 }
 895
 896                         } catch (UndefinedTransformerMethodException e) {
 897                                 throw new RuntimeException(e);
 898                         }
 899                 }
 900
 901
 902
 903                 // lat/ long /error
 904                 try {
 905                         String longitude = row.getLongitude();
 906                         String latitude = row.getLatitude();
 907                         Integer errorRadius = null;
 908                         if (StringUtils.isNotBlank(row.getErrorRadius())){
 909                                 try {
 910                                         errorRadius = Integer.valueOf(row.getErrorRadius());
 911                                 } catch (NumberFormatException e) {
 912                                         String message = "Error radius %s could not be transformed to Integer in line %d";
 913                                         message = String.format(message, row.getErrorRadius(), state.getCurrentLine());
 914                                         logger.warn(message);
 915                                 }
 916                         }
 917                         //all
 918                         facade.setExactLocationByParsing(longitude, latitude, refSys, errorRadius);
 919                 } catch (ParseException e) {
 920                         String message = "Problems when parsing exact location for line %d";
 921                         message = String.format(message, state.getCurrentLine());
 922                         logger.warn(message);
 923
 924                 }
 925
 926
 927
 928         }
 929
 930
 931         /*
 932          * Set the current Country
 933          * Search in the DB if the isoCode is known
 934          * If not, search if the country name is in the DB
 935          * If not, create a new Label with the Level Country
 936          * @param iso: the country iso code
 937          * @param fullName: the country's full name
 938          * @param app: the CDM application controller
 939          */
 940         private void handleCountry(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 941
 942                 if (StringUtils.isNotBlank(row.getIsoCountry())){
 943                         NamedArea country = getOccurrenceService().getCountryByIso(row.getIsoCountry());
 944                         if (country != null){
 945                                 facade.setCountry(country);
 946                                 return;
 947                         }
 948                 }
 949                 if (StringUtils.isNotBlank(row.getCountry())){
 950                         List<Country> countries = getOccurrenceService().getCountryByName(row.getCountry());
 951                         if (countries.size() >0){
 952                                 facade.setCountry(countries.get(0));
 953                         }else{
 954                                 UUID uuid = UUID.randomUUID();
 955                                 String label = row.getCountry();
 956                                 String text = row.getCountry();
 957                                 String labelAbbrev = null;
 958                                 NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
 959                                 NamedAreaLevel level = NamedAreaLevel.COUNTRY();
 960                                 NamedArea newCountry = this.getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level);
 961                                 facade.setCountry(newCountry);
 962                         }
 963                 }
 964         }
 965
 966         @Override
 967         protected void secondPass(SpecimenCdmExcelImportState state) {
 968                 //no second path defined yet
 969                 return;
 970         }
 971
 972
 973         @Override
 974         protected String getWorksheetName() {
 975                 return WORKSHEET_NAME;
 976         }
 977
 978         @Override
 979         protected boolean needsNomenclaturalCode() {
 980                 return false;
 981         }
 982
 983
 984         /* (non-Javadoc)
 985          * @see eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase#createDataHolderRow()
 986          */
 987         @Override
 988         protected SpecimenRow createDataHolderRow() {
 989                 return new SpecimenRow();
 990         }
 991
 992
 993
 994
 995         /* (non-Javadoc)
 996          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 997          */
 998         @Override
 999         protected boolean doCheck(SpecimenCdmExcelImportState state) {
1000                 logger.warn("Validation not yet implemented for " + this.getClass().getSimpleName());
1001                 return true;
1002         }
1003
1004
1005
1006         @Override
1007         protected boolean isIgnore(SpecimenCdmExcelImportState state) {
1008                 return !state.getConfig().isDoSpecimen();
1009         }
1010
1011
1012 }