cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenCdmExcelImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen.excel.in;
  11
  12 import java.text.ParseException;
  13 import java.util.ArrayList;
  14 import java.util.List;
  15 import java.util.UUID;
  16
  17 import org.apache.commons.lang.StringUtils;
  18 import org.apache.log4j.Logger;
  19 import org.springframework.stereotype.Component;
  20
  21 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  22 import eu.etaxonomy.cdm.api.service.config.MatchingTaxonConfigurator;
  23 import eu.etaxonomy.cdm.common.CdmUtils;
  24 import eu.etaxonomy.cdm.io.common.ICdmIO;
  25 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  26 import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase.PostfixTerm;
  27 import eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase;
  28 import eu.etaxonomy.cdm.io.specimen.excel.in.SpecimenRow.DeterminationLight;
  29 import eu.etaxonomy.cdm.model.agent.AgentBase;
  30 import eu.etaxonomy.cdm.model.agent.Person;
  31 import eu.etaxonomy.cdm.model.agent.Team;
  32 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  33 import eu.etaxonomy.cdm.model.common.Annotation;
  34 import eu.etaxonomy.cdm.model.common.AnnotationType;
  35 import eu.etaxonomy.cdm.model.common.CdmBase;
  36 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  37 import eu.etaxonomy.cdm.model.common.Language;
  38 import eu.etaxonomy.cdm.model.common.TimePeriod;
  39 import eu.etaxonomy.cdm.model.description.Feature;
  40 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  41 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  42 import eu.etaxonomy.cdm.model.location.Country;
  43 import eu.etaxonomy.cdm.model.location.NamedArea;
  44 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  45 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  46 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  47 import eu.etaxonomy.cdm.model.name.BotanicalName;
  48 import eu.etaxonomy.cdm.model.name.INonViralName;
  49 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  50 import eu.etaxonomy.cdm.model.name.NonViralName;
  51 import eu.etaxonomy.cdm.model.name.Rank;
  52 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  53 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  54 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  55 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  56 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  57 import eu.etaxonomy.cdm.model.occurrence.Collection;
  58 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  59 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  60 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  61 import eu.etaxonomy.cdm.model.reference.Reference;
  62 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  63 import eu.etaxonomy.cdm.model.taxon.Taxon;
  64 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  65 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  66 import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
  67 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  68 import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
  69 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  70 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  71
  72 /**
  73  * @author a.mueller
  74  * @created 10.05.2011
  75  * @version 1.0
  76  */
  77 @Component
  78 public class SpecimenCdmExcelImport  extends ExcelTaxonOrSpecimenImportBase<SpecimenCdmExcelImportState, SpecimenRow>  implements ICdmIO<SpecimenCdmExcelImportState> {
  79         private static final Logger logger = Logger.getLogger(SpecimenCdmExcelImport.class);
  80
  81         private static final String WORKSHEET_NAME = "Specimen";
  82
  83         private static final String BASIS_OF_RECORD_COLUMN = "(?i)(BasisOfRecord)";
  84         private static final String COUNTRY_COLUMN = "(?i)(Country)";
  85         private static final String AREA_COLUMN = "(?i)(Area)";
  86         private static final String ISO_COUNTRY_COLUMN = "(?i)(ISOCountry|CountryCode)";
  87         private static final String LOCALITY_COLUMN = "(?i)(Locality)";
  88         private static final String ALTITUDE_COLUMN = "(?i)(AbsoluteElevation|Altitude)";
  89         private static final String ALTITUDE_MAX_COLUMN = "(?i)(AbsoluteElevation|Altitude)Max(imum)?";
  90         private static final String COLLECTION_DATE_COLUMN = "(?i)(CollectionDate)";
  91         private static final String COLLECTION_DATE_END_COLUMN = "(?i)(CollectionDateEnd)";
  92         private static final String COLLECTOR_COLUMN = "(?i)(Collector)";
  93         private static final String COLLECTORS_COLUMN = "(?i)(Collectors)";
  94         private static final String PRIMARY_COLLECTOR_COLUMN = "(?i)(PrimaryCollector)";
  95         private static final String LONGITUDE_COLUMN = "(?i)(Longitude)";
  96         private static final String LATITUDE_COLUMN = "(?i)(Latitude)";
  97         private static final String REFERENCE_SYSTEM_COLUMN = "(?i)(ReferenceSystem)";
  98         private static final String ERROR_RADIUS_COLUMN = "(?i)(ErrorRadius)";
  99
 100
 101         private static final String COLLECTORS_NUMBER_COLUMN = "(?i)((Collectors|Field)Number)";
 102         private static final String ECOLOGY_COLUMN = "(?i)(Ecology|Habitat)";
 103         private static final String PLANT_DESCRIPTION_COLUMN = "(?i)(PlantDescription)";
 104         private static final String FIELD_NOTES_COLUMN = "(?i)(FieldNotes)";
 105         private static final String SEX_COLUMN = "(?i)(Sex)";
 106
 107
 108         private static final String ACCESSION_NUMBER_COLUMN = "(?i)(AccessionNumber)";
 109         private static final String BARCODE_COLUMN = "(?i)(Barcode)";
 110         private static final String COLLECTION_CODE_COLUMN = "(?i)(CollectionCode)";
 111         private static final String COLLECTION_COLUMN = "(?i)(Collection)";
 112         private static final String UNIT_NOTES_COLUMN = "(?i)((Unit)?Notes)";
 113
 114
 115         private static final String TYPE_CATEGORY_COLUMN = "(?i)(TypeCategory)";
 116         private static final String TYPIFIED_NAME_COLUMN = "(?i)(TypifiedName|TypeOf)";
 117
 118
 119         private static final String SOURCE_COLUMN = "(?i)(Source)";
 120         private static final String ID_IN_SOURCE_COLUMN = "(?i)(IdInSource)";
 121
 122
 123         private static final String DETERMINATION_AUTHOR_COLUMN = "(?i)(Author)";
 124         private static final String DETERMINATION_MODIFIER_COLUMN = "(?i)(DeterminationModifier)";
 125         private static final String DETERMINED_BY_COLUMN = "(?i)(DeterminationBy)";
 126         private static final String DETERMINED_WHEN_COLUMN = "(?i)(Det(ermination)?When)";
 127         private static final String DETERMINATION_NOTES_COLUMN = "(?i)(DeterminationNote)";
 128         private static final String EXTENSION_COLUMN = "(?i)(Ext(ension)?)";
 129
 130
 131         public SpecimenCdmExcelImport() {
 132                 super();
 133         }
 134
 135
 136
 137
 138         @Override
 139         protected void analyzeSingleValue(KeyValue keyValue, SpecimenCdmExcelImportState state) {
 140                 SpecimenRow row = state.getCurrentRow();
 141                 String value = keyValue.value;
 142                 if(keyValue.key.matches(BASIS_OF_RECORD_COLUMN)) {
 143                         row.setBasisOfRecord(value);
 144                 } else if(keyValue.key.matches(COUNTRY_COLUMN)) {
 145                         row.setCountry(value);
 146                 } else if(keyValue.key.matches(ISO_COUNTRY_COLUMN)) {
 147                         row.setIsoCountry(value);
 148                 } else if(keyValue.key.matches(LOCALITY_COLUMN)) {
 149                         row.setLocality(value);
 150                 } else if(keyValue.key.matches(FIELD_NOTES_COLUMN)) {
 151                         row.setLocality(value);
 152                 } else if(keyValue.key.matches(ALTITUDE_COLUMN)) {
 153                         row.setAltitude(value);
 154                 } else if(keyValue.key.matches(ALTITUDE_MAX_COLUMN)) {
 155                         row.setAltitudeMax(value);
 156                 } else if(keyValue.key.matches(COLLECTOR_COLUMN)) {
 157                         row.putCollector(keyValue.index, value);
 158                 } else if(keyValue.key.matches(PRIMARY_COLLECTOR_COLUMN)) {
 159                         row.setPrimaryCollector(value);
 160                 } else if(keyValue.key.matches(ECOLOGY_COLUMN)) {
 161                         row.setEcology(value);
 162                 } else if(keyValue.key.matches(PLANT_DESCRIPTION_COLUMN)) {
 163                         row.setPlantDescription(value);
 164                 } else if(keyValue.key.matches(SEX_COLUMN)) {
 165                         row.setSex(value);
 166                 } else if(keyValue.key.matches(COLLECTION_DATE_COLUMN)) {
 167                         row.setCollectingDate(value);
 168                 } else if(keyValue.key.matches(COLLECTION_DATE_END_COLUMN)) {
 169                         row.setCollectingDateEnd(value);
 170                 } else if(keyValue.key.matches(COLLECTORS_COLUMN)) {
 171                         row.setCollectors(value);
 172                 } else if(keyValue.key.matches(COLLECTOR_COLUMN)) {
 173                         row.putCollector(keyValue.index, value);
 174                 } else if(keyValue.key.matches(COLLECTORS_NUMBER_COLUMN)) {
 175                         row.setCollectorsNumber(value);
 176                 } else if(keyValue.key.matches(LONGITUDE_COLUMN)) {
 177                         row.setLongitude(value);
 178                 } else if(keyValue.key.matches(LATITUDE_COLUMN)) {
 179                         row.setLatitude(value);
 180                 } else if(keyValue.key.matches(REFERENCE_SYSTEM_COLUMN)) {
 181                         row.setReferenceSystem(value);
 182                 } else if(keyValue.key.matches(ERROR_RADIUS_COLUMN)) {
 183                         row.setErrorRadius(value);
 184                 } else if(keyValue.key.matches(AREA_COLUMN)) {
 185                         if (keyValue.postfix != null){
 186                                 row.addLeveledArea(keyValue.postfix, value);
 187                         }else{
 188                                 logger.warn("Not yet implemented");
 189                         }
 190                 } else if(keyValue.key.matches(LANGUAGE)) {
 191                         row.setLanguage(value);
 192
 193
 194                 } else if(keyValue.key.matches(ACCESSION_NUMBER_COLUMN)) {
 195                         row.setAccessionNumber(value);
 196                 } else if(keyValue.key.matches(BARCODE_COLUMN)) {
 197                         row.setBarcode(value);
 198                 } else if(keyValue.key.matches(UNIT_NOTES_COLUMN)) {
 199                         row.putUnitNote(keyValue.index, value);
 200
 201
 202                 } else if(keyValue.key.matches(FAMILY_COLUMN)) {
 203                         row.putDeterminationFamily(keyValue.index, value);
 204                 } else if(keyValue.key.matches(GENUS_COLUMN)) {
 205                         row.putDeterminationGenus(keyValue.index, value);
 206                 } else if(keyValue.key.matches(SPECIFIC_EPITHET_COLUMN)) {
 207                         row.putDeterminationSpeciesEpi(keyValue.index, value);
 208                 } else if(keyValue.key.matches(INFRASPECIFIC_EPITHET_COLUMN)) {
 209                         row.putDeterminationInfraSpeciesEpi(keyValue.index, value);
 210                 } else if(keyValue.key.matches(RANK_COLUMN)) {
 211                         row.putDeterminationRank(keyValue.index, value);
 212                 } else if(keyValue.key.matches(TAXON_UUID_COLUMN)) {
 213                         row.putDeterminationTaxonUuid(keyValue.index, value);
 214                 } else if(keyValue.key.matches(FULL_NAME_COLUMN)) {
 215                         row.putDeterminationFullName(keyValue.index, value);
 216                 } else if(keyValue.key.matches(DETERMINATION_AUTHOR_COLUMN)) {
 217                         row.putDeterminationAuthor(keyValue.index, value);
 218                 } else if(keyValue.key.matches(DETERMINATION_MODIFIER_COLUMN)) {
 219                         row.putDeterminationDeterminationModifier(keyValue.index, value);
 220                 } else if(keyValue.key.matches(DETERMINATION_NOTES_COLUMN)) {
 221                         row.putDeterminationDeterminationNotes(keyValue.index, value);
 222                 } else if(keyValue.key.matches(DETERMINED_BY_COLUMN)) {
 223                         row.putDeterminationDeterminedBy(keyValue.index, value);
 224                 } else if(keyValue.key.matches(DETERMINED_WHEN_COLUMN)) {
 225                         row.putDeterminationDeterminedWhen(keyValue.index, value);
 226
 227                 } else if(keyValue.key.matches(COLLECTION_CODE_COLUMN)) {
 228                         row.setCollectionCode(value);
 229                 } else if(keyValue.key.matches(COLLECTION_COLUMN)) {
 230                         row.setCollection(value);
 231
 232                 } else if(keyValue.key.matches(TYPE_CATEGORY_COLUMN)) {
 233                         row.putTypeCategory(keyValue.index, getSpecimenTypeStatus(state, value));
 234                 } else if(keyValue.key.matches(TYPIFIED_NAME_COLUMN)) {
 235                         row.putTypifiedName(keyValue.index, getTaxonName(state, value));
 236
 237
 238                 } else if(keyValue.key.matches(SOURCE_COLUMN)) {
 239                         row.putSourceReference(keyValue.index, getOrMakeReference(state, value) );
 240                 } else if(keyValue.key.matches(ID_IN_SOURCE_COLUMN)) {
 241                         row.putIdInSource(keyValue.index, value);
 242                 } else if(keyValue.key.matches(EXTENSION_COLUMN)) {
 243                         if (keyValue.postfix != null){
 244                                 row.addExtension(keyValue.postfix, value);
 245                         }else{
 246                                 logger.warn("Extension without postfix not yet implemented");
 247                         }
 248
 249                 }else {
 250                         state.setUnsuccessfull();
 251                         logger.error("Unexpected column header " + keyValue.originalKey);
 252                 }
 253
 254         return;
 255         }
 256
 257
 258         @Override
 259         protected void firstPass(SpecimenCdmExcelImportState state) {
 260                 SpecimenRow row = state.getCurrentRow();
 261
 262                 //basis of record
 263                 SpecimenOrObservationType type = SpecimenOrObservationType.valueOf2(row.getBasisOfRecord());
 264                 if (type == null){
 265                         String message = "%s is not a valid BasisOfRecord. 'Unknown' is used instead in line %d.";
 266                         message = String.format(message, row.getBasisOfRecord(), state.getCurrentLine());
 267                         logger.warn(message);
 268                         type = SpecimenOrObservationType.DerivedUnit;
 269                 }
 270                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(type);
 271
 272
 273                 Language lang = Language.DEFAULT();
 274                 if (StringUtils.isNotBlank(row.getLanguage())){
 275                         Language langIso = getTermService().getLanguageByIso(row.getLanguage());
 276                         if (langIso == null){
 277                                 String message = "Language could not be recognized: %s. Use default language instead. Line %d.";
 278                                 message = String.format(message, langIso, state.getCurrentLine());
 279                         }else{
 280                                 lang = langIso;
 281                         }
 282                 }
 283
 284                 //country
 285                 handleCountry(facade, row, state);
 286                 handleAreas(facade,row, state);
 287
 288                 facade.setGatheringPeriod(getTimePeriod(row.getCollectingDate(), row.getCollectingDateEnd()));
 289                 facade.setLocality(row.getLocality());
 290                 facade.setFieldNotes(row.getFieldNotes());
 291                 facade.setFieldNumber(row.getCollectorsNumber());
 292                 facade.setEcology(row.getEcology(), lang);
 293                 facade.setPlantDescription(row.getPlantDescription(), lang);
 294 //              facade.setSex(row.get)
 295                 handleExactLocation(facade, row, state);
 296                 facade.setCollector(getOrMakeAgent(state, row.getCollectors()));
 297                 facade.setPrimaryCollector(getOrMakePrimaryCollector(facade, row.getPrimaryCollector(), state));
 298                 handleAbsoluteElevation(facade, row, state);
 299
 300                 //derivedUnit
 301                 facade.setBarcode(row.getBarcode());
 302                 facade.setAccessionNumber(row.getAccessionNumber());
 303                 facade.setCollection(getOrMakeCollection(state, row.getCollectionCode(), row.getCollection()));
 304                 for (IdentifiableSource source : row.getSources()){
 305                         facade.addSource(source);
 306                 }
 307                 for (SpecimenTypeDesignation designation : row.getTypeDesignations()){
 308                         logger.warn("FIXME"); //FIXME
 309 //                      facade.innerDerivedUnit().addSpecimenTypeDesignation(designation);
 310                 }
 311                 handleDeterminations(state, row, facade);
 312                 handleExtensions(facade.innerDerivedUnit(),row, state);
 313                 for (String note : row.getUnitNotes()){
 314                         Annotation annotation = Annotation.NewInstance(note, AnnotationType.EDITORIAL(), Language.DEFAULT());
 315                         facade.addAnnotation(annotation);
 316                 }
 317
 318                 //save
 319                 getOccurrenceService().save(facade.innerDerivedUnit());
 320                 return;
 321         }
 322
 323         private void handleAbsoluteElevation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 324                 //altitude
 325
 326                 try {
 327                         String altitude = row.getAltitude();
 328                         if (StringUtils.isBlank(altitude)){
 329                                 return;
 330                         }
 331 //                      if (altitude.endsWith(".0")){
 332 //                              altitude = altitude.substring(0, altitude.length() -2);
 333 //                      }
 334                         int value = Integer.valueOf(altitude);
 335                         facade.setAbsoluteElevation(value);
 336                 } catch (NumberFormatException e) {
 337                         String message = "Absolute elevation / altitude '%s' is not an integer number in line %d";
 338                         message = String.format(message, row.getAltitude(), state.getCurrentLine());
 339                         logger.warn(message);
 340                         return;
 341                 }
 342
 343                 //max
 344
 345                 try {
 346                         String max = row.getAltitudeMax();
 347                         if (StringUtils.isBlank(max)){
 348                                 return;
 349                         }
 350 //                      if (max.endsWith(".0")){
 351 //                              max = max.substring(0, max.length() -2);
 352 //                      }
 353                         int value = Integer.valueOf(max);
 354                         //TODO avoid unequal distance
 355                         int min = facade.getAbsoluteElevation();
 356                         if ( (value - min) % 2 == 1 ){
 357                                 String message = "Altitude min-max difference ist not equal. Max reduced by 1 in line %d";
 358                                 message = String.format(message, state.getCurrentLine());
 359                                 logger.warn(message);
 360                                 value--;
 361                         }
 362                         facade.setAbsoluteElevationRange(min, value);
 363                 } catch (NumberFormatException e) {
 364                         String message = "Absolute elevation / Altitude maximum '%s' is not an integer number in line %d";
 365                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 366                         logger.warn(message);
 367                         return;
 368                 }catch (Exception e){
 369                         String message = "Error occurred when trying to write Absolute elevation / Altitude maximum '%s' in line %d";
 370                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 371                         logger.warn(message);
 372                         return;
 373
 374                 }
 375
 376
 377         }
 378
 379         private void handleAreas(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 380                 List<PostfixTerm> areas = row.getLeveledAreas();
 381
 382                 for (PostfixTerm lArea : areas){
 383                         String description = lArea.term;
 384                         String abbrev = lArea.term;
 385                         NamedAreaType type = null;
 386                         String key = lArea.postfix + "_" + lArea.term;
 387                         UUID areaUuid = state.getArea(key);
 388                         NamedAreaLevel level = state.getPostfixLevel(lArea.postfix);
 389
 390                         TermMatchMode matchMode = state.getConfig().getAreaMatchMode();
 391                         NamedArea area = getNamedArea(state, areaUuid, lArea.term, description, abbrev, type, level, null, matchMode);
 392                         facade.addCollectingArea(area);
 393                         if (areaUuid == null){
 394                                 state.putArea(key, area.getUuid());
 395                         }
 396                 }
 397         }
 398
 399
 400         /**
 401          * @param state
 402          * @param row
 403          * @param facade
 404          */
 405         private void handleDeterminations(SpecimenCdmExcelImportState state,SpecimenRow row, DerivedUnitFacade facade) {
 406                 boolean isFirstDetermination = true;
 407                 DeterminationLight commonDetermination = row.getCommonDetermination();
 408                 Taxon commonTaxon = null;
 409                 TaxonNameBase<?,?> commonName = null;
 410
 411                 boolean hasCommonTaxonInfo = (commonDetermination == null) ? false : commonDetermination.hasTaxonInformation();
 412                 if (hasCommonTaxonInfo && commonDetermination != null){
 413                         TaxonBase<?> taxonBase = null;
 414                         if (StringUtils.isNotBlank(commonDetermination.taxonUuid)){
 415                                 UUID taxonUuid = UUID.fromString(commonDetermination.taxonUuid);
 416                                 taxonBase = getTaxonService().find(taxonUuid);
 417                                 if (taxonBase == null){
 418                                         String message = "Taxon for uuid %s not found in line %d.";
 419                                         message = String.format(message, taxonUuid.toString(), state.getCurrentLine());
 420                                         logger.warn(message);
 421                                 }
 422                         }else{
 423                                 taxonBase = findBestMatchingTaxon(state, commonDetermination, state.getConfig().isCreateTaxonIfNotExists());
 424                         }
 425                         commonTaxon = getAcceptedTaxon(taxonBase);
 426                         if (taxonBase != null){
 427                                 commonName = taxonBase.getName();
 428                         }else{
 429                                 commonTaxon = createTaxonFromDetermination(state, commonDetermination);
 430                                 commonName = commonTaxon.getName();
 431                         }
 432                 }
 433
 434
 435                 for (DeterminationLight determinationLight : row.getDetermination()){
 436                         Taxon taxon;
 437                         if (! hasCommonTaxonInfo){
 438                                 taxon = findBestMatchingTaxon(state, determinationLight, state.getConfig().isCreateTaxonIfNotExists());
 439                         }else{
 440                                 taxon = commonTaxon;
 441                         }
 442                         if (taxon != null){
 443                                 getTaxonService().saveOrUpdate(taxon);
 444                                 if (state.getConfig().isMakeIndividualAssociations() && taxon != null){
 445                                         IndividualsAssociation indivAssociciation = IndividualsAssociation.NewInstance();
 446                                         DerivedUnit du = facade.innerDerivedUnit();
 447                                         indivAssociciation.setAssociatedSpecimenOrObservation(du);
 448                                         getTaxonDescription(taxon).addElement(indivAssociciation);
 449                                         Feature feature = Feature.INDIVIDUALS_ASSOCIATION();
 450                                         if (facade.getType().isPreservedSpecimen()){
 451                                                 feature = Feature.SPECIMEN();
 452                                         }else if (facade.getType().isFeatureObservation()){
 453                                                 feature = Feature.OBSERVATION();
 454                                         }
 455                                         if (state.getConfig().isUseMaterialsExaminedForIndividualsAssociations()){
 456                                                 feature = Feature.MATERIALS_EXAMINED();
 457                                         }
 458
 459                                         indivAssociciation.setFeature(feature);
 460                                 }
 461                                 if (state.getConfig().isDeterminationsAreDeterminationEvent()){
 462                                         DeterminationEvent detEvent = makeDeterminationEvent(state, determinationLight, taxon);
 463                                         detEvent.setPreferredFlag(isFirstDetermination);
 464                                         facade.addDetermination(detEvent);
 465                                 }
 466                         }
 467
 468                         if (isFirstDetermination && state.getConfig().isFirstDeterminationIsStoredUnder()){
 469                                 TaxonNameBase<?,?> name;
 470
 471                                 if (!hasCommonTaxonInfo){
 472                                         name = findBestMatchingName(state, determinationLight);
 473                                 }else{
 474                                         if (commonName == null){
 475                                                 commonName = findBestMatchingName(state, commonDetermination);
 476                                         }
 477                                         name = commonName;
 478                                 }
 479                                 if (name != null){
 480                                         facade.setStoredUnder(name);
 481                                 }
 482                         }
 483                         isFirstDetermination = false;
 484                 }
 485         }
 486
 487         private Taxon createTaxonFromDetermination( SpecimenCdmExcelImportState state, DeterminationLight commonDetermination) {
 488
 489                 //rank
 490                 Rank rank;
 491                 try {
 492                         rank = StringUtils.isBlank(commonDetermination.rank) ? null : Rank.getRankByNameOrIdInVoc(commonDetermination.rank, true);
 493                 } catch (UnknownCdmTypeException e) {
 494                         rank = null;
 495                 }
 496
 497                 //name
 498                 INonViralName name;
 499                 INonViralNameParser parser = NonViralNameParserImpl.NewInstance();
 500                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 501                 if (StringUtils.isNotBlank(commonDetermination.fullName)){
 502                         name = parser.parseFullName(commonDetermination.fullName, nc, rank);
 503                         if (StringUtils.isBlank(name.getAuthorshipCache()) && StringUtils.isNotBlank(commonDetermination.author)){
 504                                 setAuthorship(name, commonDetermination.author, parser);
 505                         }
 506                 }else{
 507                         if (nc != null){
 508                                 name = nc.getNewTaxonNameInstance(rank);
 509                         }else{
 510                                 name = TaxonNameFactory.NewNonViralInstance(rank);
 511                         }
 512                         if (StringUtils.isNotBlank(commonDetermination.genus)){
 513                                 name.setGenusOrUninomial(commonDetermination.genus);
 514                         }
 515                         if (StringUtils.isNotBlank(commonDetermination.speciesEpi)){
 516                                 name.setSpecificEpithet(commonDetermination.speciesEpi);
 517                         }
 518                         if (StringUtils.isNotBlank(commonDetermination.infraSpeciesEpi)){
 519                                 name.setInfraSpecificEpithet(commonDetermination.infraSpeciesEpi);
 520                         }
 521                         if (StringUtils.isNotBlank(commonDetermination.author)){
 522                                 setAuthorship(name, commonDetermination.author, parser);
 523                         }
 524                         //guess rank if null
 525                         if (name.getRank() == null){
 526                                 if (name.getInfraGenericEpithet() != null && name.getSpecificEpithet() == null){
 527                                         name.setRank(Rank.INFRAGENERICTAXON());
 528                                 }else if (name.getSpecificEpithet() != null && name.getInfraSpecificEpithet() == null){
 529                                         name.setRank(Rank.SPECIES());
 530                                 }else if (name.getInfraSpecificEpithet() != null){
 531                                         name.setRank(Rank.INFRASPECIFICTAXON());
 532                                 }
 533
 534                         }
 535
 536                 }
 537                 //sec
 538                 Reference sec = null;
 539                 if (StringUtils.isNotBlank(commonDetermination.determinedBy)){
 540                         sec = ReferenceFactory.newGeneric();
 541                         TeamOrPersonBase<?> determinedBy;
 542                         BotanicalName dummyName = TaxonNameBase.NewBotanicalInstance(Rank.SPECIES());
 543                         try {
 544                                 parser.parseAuthors(dummyName, commonDetermination.determinedBy);
 545                                 determinedBy = dummyName.getCombinationAuthorship();
 546                         } catch (StringNotParsableException e) {
 547                                 determinedBy = Team.NewTitledInstance(commonDetermination.determinedBy, commonDetermination.determinedBy);
 548                         }
 549                         sec.setAuthorship(determinedBy);
 550                 }
 551
 552                 //taxon
 553                 Taxon taxon = Taxon.NewInstance(name, sec);
 554
 555                 if (StringUtils.isNotBlank(commonDetermination.family)){
 556                         if (name.getRank() == null || name.getRank().isLower(Rank.FAMILY()) ){
 557                                 logger.warn("Family taxon could not be created");
 558                         }
 559                 }
 560
 561                 //return
 562                 return taxon;
 563
 564         }
 565
 566
 567
 568
 569         private void setAuthorship(INonViralName name, String author, INonViralNameParser<NonViralName> parser) {
 570                 if (name instanceof BotanicalName || name instanceof ZoologicalName){
 571                         try {
 572                                 parser.parseAuthors(name, author);
 573                         } catch (StringNotParsableException e) {
 574                                 name.setAuthorshipCache(author);
 575                         }
 576                 }else{
 577                         name.setAuthorshipCache(author);
 578                 }
 579         }
 580
 581
 582
 583         /**
 584          * This method tries to find the best matching taxon depending on the import configuration,
 585          * the taxon name information and the concept information available.
 586          *
 587          *
 588          * @param state
 589          * @param determinationLight
 590          * @param createIfNotExists
 591          * @return
 592          */
 593         private Taxon findBestMatchingTaxon(SpecimenCdmExcelImportState state, DeterminationLight determinationLight, boolean createIfNotExists) {
 594                 NonViralName<?> name = makeTaxonName(state, determinationLight);
 595
 596                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 597
 598                 if (! StringUtils.isBlank(titleCache)){
 599                         MatchingTaxonConfigurator matchConfigurator = MatchingTaxonConfigurator.NewInstance();
 600                         matchConfigurator.setTaxonNameTitle(titleCache);
 601                         matchConfigurator.setIncludeSynonyms(false);
 602                         Taxon taxon = getTaxonService().findBestMatchingTaxon(matchConfigurator);
 603
 604                         if(taxon == null && createIfNotExists){
 605                                 logger.info("creating new Taxon from TaxonName '" + titleCache+"'");
 606                                 UUID secUuid = null; //TODO
 607                                 Reference sec = null;
 608                                 if (secUuid != null){
 609                                         sec = getReferenceService().find(secUuid);
 610                                 }
 611                                 taxon = Taxon.NewInstance(name, sec);
 612                         }else if (taxon == null){
 613                                 String message = "Taxon '%s' not found in line %d";
 614                                 message = String.format(message, titleCache, state.getCurrentLine());
 615                                 logger.warn(message);
 616                         }
 617                         return taxon;
 618                 }else {
 619                         return null;
 620                 }
 621         }
 622
 623         /**
 624          * @param state
 625          * @param determinationLight
 626          * @param name
 627          * @return
 628          */
 629         private String makeSearchNameTitleCache(SpecimenCdmExcelImportState state, DeterminationLight determinationLight,
 630                                 NonViralName<?> name) {
 631                 String titleCache = determinationLight.fullName;
 632                 if (! state.getConfig().isPreferNameCache() || StringUtils.isBlank(titleCache) ){
 633                         String computedTitleCache = name.getTitleCache();
 634                         if (StringUtils.isNotBlank(computedTitleCache)){
 635                                 titleCache = computedTitleCache;
 636                         }
 637
 638                 }
 639                 return titleCache;
 640         }
 641
 642         /**
 643          * @param state
 644          * @param determinationLight
 645          * @return
 646          */
 647         private NonViralName<?> makeTaxonName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 648                 NonViralName<?> name = TaxonNameFactory.NewNonViralInstance(null);
 649                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 650                 if (nc != null){
 651                         name = (NonViralName<?>)nc.getNewTaxonNameInstance(null);
 652                 }
 653                 name.setGenusOrUninomial(determinationLight.genus);
 654                 name.setSpecificEpithet(determinationLight.speciesEpi);
 655                 name.setInfraSpecificEpithet(determinationLight.infraSpeciesEpi);
 656
 657                 //FIXME bracketAuthors and teams not yet implemented!!!
 658                 List<String> authors = new ArrayList<String>();
 659                 if (StringUtils.isNotBlank(determinationLight.author)){
 660                         authors.add(determinationLight.author);
 661                 }
 662                 TeamOrPersonBase<?> agent = getOrMakeAgent(state, authors);
 663                 name.setCombinationAuthorship(agent);
 664
 665                 try {
 666                         if (StringUtils.isNotBlank(determinationLight.rank) ){
 667                                 name.setRank(Rank.getRankByNameOrIdInVoc(determinationLight.rank, nc, true));
 668                         }
 669                 } catch (UnknownCdmTypeException e) {
 670                         String message = "Rank not found: %s: ";
 671                         message = String.format(message, determinationLight.rank);
 672                         logger.warn(message);
 673                 }
 674                 if (StringUtils.isBlank(name.getInfraSpecificEpithet()) && StringUtils.isNotBlank(name.getSpecificEpithet() )){
 675                         name.setRank(Rank.SPECIES());
 676                 }
 677                 if (StringUtils.isBlank(name.getSpecificEpithet()) && StringUtils.isNotBlank(name.getGenusOrUninomial() )){
 678                         name.setRank(Rank.SPECIES());
 679                 }
 680                 if (StringUtils.isBlank(name.getTitleCache())){
 681                         //TODO test
 682                         name.setTitleCache(determinationLight.fullName, true);
 683                 }
 684                 return name;
 685         }
 686
 687         private TaxonNameBase findBestMatchingName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 688
 689                 NonViralName<?> name = makeTaxonName(state, determinationLight);
 690                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 691
 692                 //TODO
 693                 List<TaxonNameBase> matchingNames = getNameService().findByName(null, titleCache, MatchMode.EXACT, null, null, null, null, null).getRecords();
 694                 if (matchingNames.size() > 0){
 695                         return matchingNames.get(0);
 696                 } else if (matchingNames.size() > 0){
 697                         logger.warn("Get best matching taxon name not yet fully implemeted for specimen import");
 698                         return matchingNames.get(0);
 699                 }else{
 700                         return null;
 701                 }
 702
 703         }
 704
 705
 706         private DeterminationEvent makeDeterminationEvent(SpecimenCdmExcelImportState state, DeterminationLight determination, Taxon taxon) {
 707                 DeterminationEvent event = DeterminationEvent.NewInstance();
 708                 //taxon
 709                 event.setTaxon(taxon);
 710
 711                 //date
 712                 TimePeriod date = TimePeriodParser.parseString(determination.determinedWhen);
 713                 event.setTimeperiod(date);
 714                 //by
 715                 //FIXME bracketAuthors and teams not yet implemented!!!
 716                 List<String> authors = new ArrayList<String>();
 717                 if (StringUtils.isNotBlank(determination.determinedBy)){
 718                         authors.add(determination.determinedBy);
 719                 }
 720                 TeamOrPersonBase<?> actor = getOrMakeAgent(state, authors);
 721                 TeamOrPersonBase<?> secAuthor = taxon.getSec() == null ? null : taxon.getSec().getAuthorship();
 722                 if (actor != null && secAuthor != null && secAuthor.getTitleCache().equals(actor.getTitleCache()) && secAuthor.getNomenclaturalTitle().equals(actor.getNomenclaturalTitle())) {
 723                         actor = secAuthor;
 724                 }
 725
 726                 event.setActor(actor);
 727
 728                 //TODO
 729                 if (StringUtils.isNotBlank(determination.modifier)){
 730                         logger.warn("DeterminationModifiers not yet implemented for specimen import");
 731                 }
 732 //              DeterminationModifier modifier = DeterminationModifier.NewInstance(term, label, labelAbbrev);
 733 //              determination.modifier;
 734                 //notes
 735                 if (StringUtils.isNotEmpty(determination.notes)){
 736                         Annotation annotation = Annotation.NewInstance(determination.notes, AnnotationType.EDITORIAL(), Language.DEFAULT());
 737                         event.addAnnotation(annotation);
 738                 }
 739
 740                 return event;
 741         }
 742
 743         private TaxonDescription getTaxonDescription(Taxon taxon) {
 744                 TaxonDescription desc = this.getTaxonDescription(taxon, ! IMAGE_GALLERY, CREATE);
 745                 return desc;
 746         }
 747
 748         private TeamOrPersonBase<?> getOrMakeAgent(SpecimenCdmExcelImportState state, List<String> agents) {
 749                 if (agents.size() == 0){
 750                         return null;
 751                 }else if (agents.size() == 1){
 752                         return getOrMakePerson(state, agents.get(0));
 753                 }else{
 754                         return getOrMakeTeam(state, agents);
 755                 }
 756         }
 757
 758         private Person getOrMakePrimaryCollector(DerivedUnitFacade facade, String primaryCollector, SpecimenCdmExcelImportState state) {
 759                 if (StringUtils.isBlank(primaryCollector)){
 760                         return null;
 761                 }
 762                 AgentBase<?> collector = facade.getCollector();
 763                 List<Person> collectors = new ArrayList<Person>();
 764                 if (collector.isInstanceOf(Team.class) ){
 765                         Team team = CdmBase.deproxy(collector, Team.class);
 766                         collectors.addAll(team.getTeamMembers());
 767                 }else if (collector.isInstanceOf(Person.class)){
 768                         collectors.add(CdmBase.deproxy(collector, Person.class));
 769                 }else{
 770                         throw new IllegalStateException("Unknown subclass of agentbase: " + collector.getClass().getName() );
 771                 }
 772                 for (Person person :collectors){
 773                         if (primaryCollector.equalsIgnoreCase(person.getTitleCache())){
 774                                 return person;
 775                         }
 776                         if (primaryCollector.equalsIgnoreCase(person.getNomenclaturalTitle())){
 777                                 return person;
 778                         }
 779                 }
 780                 String message = "Primary Agent '%s' could not be determined in collector(s) in line %d";
 781                 message = String.format(message, primaryCollector, state.getCurrentLine());
 782                 logger.warn(message);
 783                 return null;
 784         }
 785
 786         private Team getOrMakeTeam(SpecimenCdmExcelImportState state, List<String> agents) {
 787                 String key = CdmUtils.concat("_", agents.toArray(new String[0]));
 788
 789                 Team result = state.getTeam(key);
 790                 if (result == null){
 791                         result = Team.NewInstance();
 792                         for (String member : agents){
 793                                 Person person = getOrMakePerson(state, member);
 794                                 result.addTeamMember(person);
 795                         }
 796                         state.putTeam(key, result);
 797                 }
 798                 return result;
 799         }
 800
 801         private Person getOrMakePerson(SpecimenCdmExcelImportState state, String value) {
 802                 Person result = state.getPerson(value);
 803                 if (result == null){
 804                         result = Person.NewInstance();
 805                         result.setTitleCache(value, true);
 806                         state.putPerson(value, result);
 807                 }
 808                 return result;
 809         }
 810
 811         private Reference getOrMakeReference(SpecimenCdmExcelImportState state, String value) {
 812                 Reference result = state.getReference(value);
 813                 if (result == null){
 814                         result = ReferenceFactory.newGeneric();
 815                         result.setTitleCache(value, true);
 816                         state.putReference(value, result);
 817                 }
 818                 return result;
 819         }
 820
 821
 822
 823         private Collection getOrMakeCollection(SpecimenCdmExcelImportState state, String collectionCode, String collectionString) {
 824                 Collection result = state.getCollection(collectionCode);
 825                 if (result == null){
 826                         result = Collection.NewInstance();
 827                         result.setCode(collectionCode);
 828                         result.setName(collectionString);
 829                         state.putCollection(collectionCode, result);
 830                 }
 831                 return result;
 832         }
 833
 834
 835         private TaxonNameBase<?, ?> getTaxonName(SpecimenCdmExcelImportState state, String name) {
 836                 TaxonNameBase<?,?> result = null;
 837                 result = state.getName(name);
 838                 if (result != null){
 839                         return result;
 840                 }
 841                 List<TaxonNameBase> list = getNameService().findByTitle(null, name, null, null, null, null, null, null).getRecords();
 842                 //TODO better strategy to find best name, e.g. depending on the classification it is used in
 843                 if (! list.isEmpty()){
 844                         result = list.get(0);
 845                 }
 846                 if (result == null){
 847                         NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
 848                         NomenclaturalCode code = state.getConfig().getNomenclaturalCode();
 849                         result = parser.parseFullName(name, code, null);
 850
 851                 }
 852                 if (result != null){
 853                         state.putName(name, result);
 854                 }
 855                 return result;
 856         }
 857
 858         private SpecimenTypeDesignationStatus getSpecimenTypeStatus(SpecimenCdmExcelImportState state, String key)  {
 859                 SpecimenTypeDesignationStatus result = null;
 860                 try {
 861                         result = state.getTransformer().getSpecimenTypeDesignationStatusByKey(key);
 862                         if (result == null){
 863                                 String message = "Type status not recognized for %s in line %d";
 864                                 message = String.format(message, key, state.getCurrentLine());
 865                                 logger.warn(message);
 866                         }
 867                         return result;
 868                 } catch (UndefinedTransformerMethodException e) {
 869                         throw new RuntimeException("getSpecimenTypeDesignationStatusByKey not yet implemented");
 870                 }
 871
 872
 873         }
 874
 875
 876         private void handleExactLocation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 877
 878                 //reference system
 879                 ReferenceSystem refSys = null;
 880                 if (StringUtils.isNotBlank(row.getReferenceSystem())){
 881                         String strRefSys = row.getReferenceSystem().trim().replaceAll("\\s", "");
 882                         UUID refUuid;
 883                         try {
 884                                 refSys = state.getTransformer().getReferenceSystemByKey(strRefSys);
 885                                 if (refSys == null){
 886                                         //TODO we still need user defined Reference Systems here
 887                                         refUuid = state.getTransformer().getReferenceSystemUuid(strRefSys);
 888                                         if (refUuid == null){
 889                                                 String message = "Unknown reference system %s in line %d";
 890                                                 message = String.format(message, strRefSys, state.getCurrentLine());
 891                                                 logger.warn(message);
 892                                         }
 893                                         refSys = getReferenceSystem(state, refUuid, strRefSys, strRefSys, strRefSys, null);
 894                                 }
 895
 896                         } catch (UndefinedTransformerMethodException e) {
 897                                 throw new RuntimeException(e);
 898                         }
 899                 }
 900
 901
 902
 903                 // lat/ long /error
 904                 try {
 905                         String longitude = row.getLongitude();
 906                         String latitude = row.getLatitude();
 907                         Integer errorRadius = null;
 908                         if (StringUtils.isNotBlank(row.getErrorRadius())){
 909                                 try {
 910                                         errorRadius = Integer.valueOf(row.getErrorRadius());
 911                                 } catch (NumberFormatException e) {
 912                                         String message = "Error radius %s could not be transformed to Integer in line %d";
 913                                         message = String.format(message, row.getErrorRadius(), state.getCurrentLine());
 914                                         logger.warn(message);
 915                                 }
 916                         }
 917                         //all
 918                         facade.setExactLocationByParsing(longitude, latitude, refSys, errorRadius);
 919                 } catch (ParseException e) {
 920                         String message = "Problems when parsing exact location for line %d";
 921                         message = String.format(message, state.getCurrentLine());
 922                         logger.warn(message);
 923
 924                 }
 925
 926
 927
 928         }
 929
 930
 931         /*
 932          * Set the current Country
 933          * Search in the DB if the isoCode is known
 934          * If not, search if the country name is in the DB
 935          * If not, create a new Label with the Level Country
 936          * @param iso: the country iso code
 937          * @param fullName: the country's full name
 938          * @param app: the CDM application controller
 939          */
 940         private void handleCountry(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 941
 942                 if (StringUtils.isNotBlank(row.getIsoCountry())){
 943                         NamedArea country = getOccurrenceService().getCountryByIso(row.getIsoCountry());
 944                         if (country != null){
 945                                 facade.setCountry(country);
 946                                 return;
 947                         }
 948                 }
 949                 if (StringUtils.isNotBlank(row.getCountry())){
 950                         List<Country> countries = getOccurrenceService().getCountryByName(row.getCountry());
 951                         if (countries.size() >0){
 952                                 facade.setCountry(countries.get(0));
 953                         }else{
 954                                 UUID uuid = UUID.randomUUID();
 955                                 String label = row.getCountry();
 956                                 String text = row.getCountry();
 957                                 String labelAbbrev = null;
 958                                 NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
 959                                 NamedAreaLevel level = NamedAreaLevel.COUNTRY();
 960                                 NamedArea newCountry = this.getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level);
 961                                 facade.setCountry(newCountry);
 962                         }
 963                 }
 964         }
 965
 966         @Override
 967         protected void secondPass(SpecimenCdmExcelImportState state) {
 968                 //no second path defined yet
 969                 return;
 970         }
 971
 972
 973         @Override
 974         protected String getWorksheetName() {
 975                 return WORKSHEET_NAME;
 976         }
 977
 978         @Override
 979         protected boolean needsNomenclaturalCode() {
 980                 return false;
 981         }
 982
 983
 984         /* (non-Javadoc)
 985          * @see eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase#createDataHolderRow()
 986          */
 987         @Override
 988         protected SpecimenRow createDataHolderRow() {
 989                 return new SpecimenRow();
 990         }
 991
 992
 993
 994
 995         /* (non-Javadoc)
 996          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 997          */
 998         @Override
 999         protected boolean doCheck(SpecimenCdmExcelImportState state) {
1000                 logger.warn("Validation not yet implemented for " + this.getClass().getSimpleName());
1001                 return true;
1002         }
1003
1004
1005
1006         @Override
1007         protected boolean isIgnore(SpecimenCdmExcelImportState state) {
1008                 return !state.getConfig().isDoSpecimen();
1009         }
1010
1011
1012 }