cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenCdmExcelImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen.excel.in;
  11
  12 import java.text.ParseException;
  13 import java.util.ArrayList;
  14 import java.util.List;
  15 import java.util.UUID;
  16
  17 import org.apache.commons.lang.StringUtils;
  18 import org.apache.log4j.Logger;
  19 import org.springframework.stereotype.Component;
  20
  21 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  22 import eu.etaxonomy.cdm.api.service.config.MatchingTaxonConfigurator;
  23 import eu.etaxonomy.cdm.common.CdmUtils;
  24 import eu.etaxonomy.cdm.io.common.ICdmIO;
  25 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  26 import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase.PostfixTerm;
  27 import eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase;
  28 import eu.etaxonomy.cdm.io.specimen.excel.in.SpecimenRow.DeterminationLight;
  29 import eu.etaxonomy.cdm.model.agent.AgentBase;
  30 import eu.etaxonomy.cdm.model.agent.Person;
  31 import eu.etaxonomy.cdm.model.agent.Team;
  32 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  33 import eu.etaxonomy.cdm.model.common.Annotation;
  34 import eu.etaxonomy.cdm.model.common.AnnotationType;
  35 import eu.etaxonomy.cdm.model.common.CdmBase;
  36 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  37 import eu.etaxonomy.cdm.model.common.Language;
  38 import eu.etaxonomy.cdm.model.common.TimePeriod;
  39 import eu.etaxonomy.cdm.model.description.Feature;
  40 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  41 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  42 import eu.etaxonomy.cdm.model.location.NamedArea;
  43 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  44 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  45 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  46 import eu.etaxonomy.cdm.model.location.Country;
  47 import eu.etaxonomy.cdm.model.name.BotanicalName;
  48 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  49 import eu.etaxonomy.cdm.model.name.NonViralName;
  50 import eu.etaxonomy.cdm.model.name.Rank;
  51 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  52 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  53 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  54 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  55 import eu.etaxonomy.cdm.model.occurrence.Collection;
  56 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  57 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  58 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  59 import eu.etaxonomy.cdm.model.reference.Reference;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  61 import eu.etaxonomy.cdm.model.taxon.Taxon;
  62 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  63 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  64 import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
  65 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  66 import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
  67 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  68 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  69
  70 /**
  71  * @author a.mueller
  72  * @created 10.05.2011
  73  * @version 1.0
  74  */
  75 @Component
  76 public class SpecimenCdmExcelImport  extends ExcelTaxonOrSpecimenImportBase<SpecimenCdmExcelImportState, SpecimenRow>  implements ICdmIO<SpecimenCdmExcelImportState> {
  77         private static final Logger logger = Logger.getLogger(SpecimenCdmExcelImport.class);
  78
  79         private static final String WORKSHEET_NAME = "Specimen";
  80
  81         private static final String BASIS_OF_RECORD_COLUMN = "(?i)(BasisOfRecord)";
  82         private static final String COUNTRY_COLUMN = "(?i)(Country)";
  83         private static final String AREA_COLUMN = "(?i)(Area)";
  84         private static final String ISO_COUNTRY_COLUMN = "(?i)(ISOCountry|CountryCode)";
  85         private static final String LOCALITY_COLUMN = "(?i)(Locality)";
  86         private static final String ALTITUDE_COLUMN = "(?i)(AbsoluteElevation|Altitude)";
  87         private static final String ALTITUDE_MAX_COLUMN = "(?i)(AbsoluteElevation|Altitude)Max(imum)?";
  88         private static final String COLLECTION_DATE_COLUMN = "(?i)(CollectionDate)";
  89         private static final String COLLECTION_DATE_END_COLUMN = "(?i)(CollectionDateEnd)";
  90         private static final String COLLECTOR_COLUMN = "(?i)(Collector)";
  91         private static final String COLLECTORS_COLUMN = "(?i)(Collectors)";
  92         private static final String PRIMARY_COLLECTOR_COLUMN = "(?i)(PrimaryCollector)";
  93         private static final String LONGITUDE_COLUMN = "(?i)(Longitude)";
  94         private static final String LATITUDE_COLUMN = "(?i)(Latitude)";
  95         private static final String REFERENCE_SYSTEM_COLUMN = "(?i)(ReferenceSystem)";
  96         private static final String ERROR_RADIUS_COLUMN = "(?i)(ErrorRadius)";
  97
  98
  99         private static final String COLLECTORS_NUMBER_COLUMN = "(?i)((Collectors|Field)Number)";
 100         private static final String ECOLOGY_COLUMN = "(?i)(Ecology|Habitat)";
 101         private static final String PLANT_DESCRIPTION_COLUMN = "(?i)(PlantDescription)";
 102         private static final String FIELD_NOTES_COLUMN = "(?i)(FieldNotes)";
 103         private static final String SEX_COLUMN = "(?i)(Sex)";
 104
 105
 106         private static final String ACCESSION_NUMBER_COLUMN = "(?i)(AccessionNumber)";
 107         private static final String BARCODE_COLUMN = "(?i)(Barcode)";
 108         private static final String COLLECTION_CODE_COLUMN = "(?i)(CollectionCode)";
 109         private static final String COLLECTION_COLUMN = "(?i)(Collection)";
 110         private static final String UNIT_NOTES_COLUMN = "(?i)((Unit)?Notes)";
 111
 112
 113         private static final String TYPE_CATEGORY_COLUMN = "(?i)(TypeCategory)";
 114         private static final String TYPIFIED_NAME_COLUMN = "(?i)(TypifiedName|TypeOf)";
 115
 116
 117         private static final String SOURCE_COLUMN = "(?i)(Source)";
 118         private static final String ID_IN_SOURCE_COLUMN = "(?i)(IdInSource)";
 119
 120
 121         private static final String DETERMINATION_AUTHOR_COLUMN = "(?i)(Author)";
 122         private static final String DETERMINATION_MODIFIER_COLUMN = "(?i)(DeterminationModifier)";
 123         private static final String DETERMINED_BY_COLUMN = "(?i)(DeterminationBy)";
 124         private static final String DETERMINED_WHEN_COLUMN = "(?i)(Det(ermination)?When)";
 125         private static final String DETERMINATION_NOTES_COLUMN = "(?i)(DeterminationNote)";
 126         private static final String EXTENSION_COLUMN = "(?i)(Ext(ension)?)";
 127
 128
 129         public SpecimenCdmExcelImport() {
 130                 super();
 131         }
 132
 133
 134
 135
 136         @Override
 137         protected void analyzeSingleValue(KeyValue keyValue, SpecimenCdmExcelImportState state) {
 138                 SpecimenRow row = state.getCurrentRow();
 139                 String value = keyValue.value;
 140                 if(keyValue.key.matches(BASIS_OF_RECORD_COLUMN)) {
 141                         row.setBasisOfRecord(value);
 142                 } else if(keyValue.key.matches(COUNTRY_COLUMN)) {
 143                         row.setCountry(value);
 144                 } else if(keyValue.key.matches(ISO_COUNTRY_COLUMN)) {
 145                         row.setIsoCountry(value);
 146                 } else if(keyValue.key.matches(LOCALITY_COLUMN)) {
 147                         row.setLocality(value);
 148                 } else if(keyValue.key.matches(FIELD_NOTES_COLUMN)) {
 149                         row.setLocality(value);
 150                 } else if(keyValue.key.matches(ALTITUDE_COLUMN)) {
 151                         row.setAltitude(value);
 152                 } else if(keyValue.key.matches(ALTITUDE_MAX_COLUMN)) {
 153                         row.setAltitudeMax(value);
 154                 } else if(keyValue.key.matches(COLLECTOR_COLUMN)) {
 155                         row.putCollector(keyValue.index, value);
 156                 } else if(keyValue.key.matches(PRIMARY_COLLECTOR_COLUMN)) {
 157                         row.setPrimaryCollector(value);
 158                 } else if(keyValue.key.matches(ECOLOGY_COLUMN)) {
 159                         row.setEcology(value);
 160                 } else if(keyValue.key.matches(PLANT_DESCRIPTION_COLUMN)) {
 161                         row.setPlantDescription(value);
 162                 } else if(keyValue.key.matches(SEX_COLUMN)) {
 163                         row.setSex(value);
 164                 } else if(keyValue.key.matches(COLLECTION_DATE_COLUMN)) {
 165                         row.setCollectingDate(value);
 166                 } else if(keyValue.key.matches(COLLECTION_DATE_END_COLUMN)) {
 167                         row.setCollectingDateEnd(value);
 168                 } else if(keyValue.key.matches(COLLECTORS_COLUMN)) {
 169                         row.setCollectors(value);
 170                 } else if(keyValue.key.matches(COLLECTOR_COLUMN)) {
 171                         row.putCollector(keyValue.index, value);
 172                 } else if(keyValue.key.matches(COLLECTORS_NUMBER_COLUMN)) {
 173                         row.setCollectorsNumber(value);
 174                 } else if(keyValue.key.matches(LONGITUDE_COLUMN)) {
 175                         row.setLongitude(value);
 176                 } else if(keyValue.key.matches(LATITUDE_COLUMN)) {
 177                         row.setLatitude(value);
 178                 } else if(keyValue.key.matches(REFERENCE_SYSTEM_COLUMN)) {
 179                         row.setReferenceSystem(value);
 180                 } else if(keyValue.key.matches(ERROR_RADIUS_COLUMN)) {
 181                         row.setErrorRadius(value);
 182                 } else if(keyValue.key.matches(AREA_COLUMN)) {
 183                         if (keyValue.postfix != null){
 184                                 row.addLeveledArea(keyValue.postfix, value);
 185                         }else{
 186                                 logger.warn("Not yet implemented");
 187                         }
 188
 189
 190
 191                 } else if(keyValue.key.matches(ACCESSION_NUMBER_COLUMN)) {
 192                         row.setLocality(value);
 193                 } else if(keyValue.key.matches(BARCODE_COLUMN)) {
 194                         row.setBarcode(value);
 195                 } else if(keyValue.key.matches(UNIT_NOTES_COLUMN)) {
 196                         row.putUnitNote(keyValue.index, value);
 197
 198
 199                 } else if(keyValue.key.matches(FAMILY_COLUMN)) {
 200                         row.putDeterminationFamily(keyValue.index, value);
 201                 } else if(keyValue.key.matches(GENUS_COLUMN)) {
 202                         row.putDeterminationGenus(keyValue.index, value);
 203                 } else if(keyValue.key.matches(SPECIFIC_EPITHET_COLUMN)) {
 204                         row.putDeterminationSpeciesEpi(keyValue.index, value);
 205                 } else if(keyValue.key.matches(INFRASPECIFIC_EPITHET_COLUMN)) {
 206                         row.putDeterminationInfraSpeciesEpi(keyValue.index, value);
 207                 } else if(keyValue.key.matches(RANK_COLUMN)) {
 208                         row.putDeterminationRank(keyValue.index, value);
 209                 } else if(keyValue.key.matches(TAXON_UUID_COLUMN)) {
 210                         row.putDeterminationTaxonUuid(keyValue.index, value);
 211                 } else if(keyValue.key.matches(FULL_NAME_COLUMN)) {
 212                         row.putDeterminationFullName(keyValue.index, value);
 213                 } else if(keyValue.key.matches(DETERMINATION_AUTHOR_COLUMN)) {
 214                         row.putDeterminationAuthor(keyValue.index, value);
 215                 } else if(keyValue.key.matches(DETERMINATION_MODIFIER_COLUMN)) {
 216                         row.putDeterminationDeterminationModifier(keyValue.index, value);
 217                 } else if(keyValue.key.matches(DETERMINATION_NOTES_COLUMN)) {
 218                         row.putDeterminationDeterminationNotes(keyValue.index, value);
 219                 } else if(keyValue.key.matches(DETERMINED_BY_COLUMN)) {
 220                         row.putDeterminationDeterminedBy(keyValue.index, value);
 221                 } else if(keyValue.key.matches(DETERMINED_WHEN_COLUMN)) {
 222                         row.putDeterminationDeterminedWhen(keyValue.index, value);
 223
 224                 } else if(keyValue.key.matches(COLLECTION_CODE_COLUMN)) {
 225                         row.setCollectionCode(value);
 226                 } else if(keyValue.key.matches(COLLECTION_COLUMN)) {
 227                         row.setCollection(value);
 228
 229                 } else if(keyValue.key.matches(TYPE_CATEGORY_COLUMN)) {
 230                         row.putTypeCategory(keyValue.index, getSpecimenTypeStatus(state, value));
 231                 } else if(keyValue.key.matches(TYPIFIED_NAME_COLUMN)) {
 232                         row.putTypifiedName(keyValue.index, getTaxonName(state, value));
 233
 234
 235                 } else if(keyValue.key.matches(SOURCE_COLUMN)) {
 236                         row.putSourceReference(keyValue.index, getOrMakeReference(state, value) );
 237                 } else if(keyValue.key.matches(ID_IN_SOURCE_COLUMN)) {
 238                         row.putIdInSource(keyValue.index, value);
 239                 } else if(keyValue.key.matches(EXTENSION_COLUMN)) {
 240                         if (keyValue.postfix != null){
 241                                 row.addExtension(keyValue.postfix, value);
 242                         }else{
 243                                 logger.warn("Extension without postfix not yet implemented");
 244                         }
 245
 246                 }else {
 247                         state.setUnsuccessfull();
 248                         logger.error("Unexpected column header " + keyValue.originalKey);
 249                 }
 250
 251         return;
 252         }
 253
 254
 255         @Override
 256         protected void firstPass(SpecimenCdmExcelImportState state) {
 257                 SpecimenRow row = state.getCurrentRow();
 258
 259                 //basis of record
 260                 SpecimenOrObservationType type = SpecimenOrObservationType.valueOf2(row.getBasisOfRecord());
 261                 if (type == null){
 262                         String message = "%s is not a valid BasisOfRecord. 'Unknown' is used instead in line %d.";
 263                         message = String.format(message, row.getBasisOfRecord(), state.getCurrentLine());
 264                         logger.warn(message);
 265                         type = SpecimenOrObservationType.DerivedUnit;
 266                 }
 267                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(type);
 268
 269                 //country
 270                 handleCountry(facade, row, state);
 271                 handleAreas(facade,row, state);
 272
 273                 facade.setGatheringPeriod(getTimePeriod(row.getCollectingDate(), row.getCollectingDateEnd()));
 274                 facade.setLocality(row.getLocality());
 275                 facade.setFieldNotes(row.getFieldNotes());
 276                 facade.setFieldNumber(row.getCollectorsNumber());
 277                 facade.setEcology(row.getEcology());
 278                 facade.setPlantDescription(row.getPlantDescription());
 279 //              facade.setSex(row.get)
 280                 handleExactLocation(facade, row, state);
 281                 facade.setCollector(getOrMakeAgent(state, row.getCollectors()));
 282                 facade.setPrimaryCollector(getOrMakePrimaryCollector(facade, row.getPrimaryCollector(), state));
 283                 handleAbsoluteElevation(facade, row, state);
 284
 285
 286                 //derivedUnit
 287                 facade.setBarcode(row.getBarcode());
 288                 facade.setAccessionNumber(row.getAccessionNumber());
 289                 facade.setCollection(getOrMakeCollection(state, row.getCollectionCode(), row.getCollection()));
 290                 for (IdentifiableSource source : row.getSources()){
 291                         facade.addSource(source);
 292                 }
 293                 for (SpecimenTypeDesignation designation : row.getTypeDesignations()){
 294                         logger.warn("FIXME"); //FIXME
 295 //                      facade.innerDerivedUnit().addSpecimenTypeDesignation(designation);
 296                 }
 297                 handleDeterminations(state, row, facade);
 298                 handleExtensions(facade.innerDerivedUnit(),row, state);
 299                 for (String note : row.getUnitNotes()){
 300                         Annotation annotation = Annotation.NewInstance(note, AnnotationType.EDITORIAL(), Language.DEFAULT());
 301                         facade.addAnnotation(annotation);
 302                 }
 303
 304                 //save
 305                 getOccurrenceService().save(facade.innerDerivedUnit());
 306                 return;
 307         }
 308
 309         private void handleAbsoluteElevation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 310                 //altitude
 311
 312                 try {
 313                         String altitude = row.getAltitude();
 314                         if (StringUtils.isBlank(altitude)){
 315                                 return;
 316                         }
 317 //                      if (altitude.endsWith(".0")){
 318 //                              altitude = altitude.substring(0, altitude.length() -2);
 319 //                      }
 320                         int value = Integer.valueOf(altitude);
 321                         facade.setAbsoluteElevation(value);
 322                 } catch (NumberFormatException e) {
 323                         String message = "Absolute elevation / altitude '%s' is not an integer number in line %d";
 324                         message = String.format(message, row.getAltitude(), state.getCurrentLine());
 325                         logger.warn(message);
 326                         return;
 327                 }
 328
 329                 //max
 330
 331                 try {
 332                         String max = row.getAltitudeMax();
 333                         if (StringUtils.isBlank(max)){
 334                                 return;
 335                         }
 336 //                      if (max.endsWith(".0")){
 337 //                              max = max.substring(0, max.length() -2);
 338 //                      }
 339                         int value = Integer.valueOf(max);
 340                         //TODO avoid unequal distance
 341                         int min = facade.getAbsoluteElevation();
 342                         if ( (value - min) % 2 == 1 ){
 343                                 String message = "Altitude min-max difference ist not equal. Max reduced by 1 in line %d";
 344                                 message = String.format(message, state.getCurrentLine());
 345                                 logger.warn(message);
 346                                 value--;
 347                         }
 348                         facade.setAbsoluteElevationRange(min, value);
 349                 } catch (NumberFormatException e) {
 350                         String message = "Absolute elevation / Altitude maximum '%s' is not an integer number in line %d";
 351                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 352                         logger.warn(message);
 353                         return;
 354                 }catch (Exception e){
 355                         String message = "Error occurred when trying to write Absolute elevation / Altitude maximum '%s' in line %d";
 356                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 357                         logger.warn(message);
 358                         return;
 359
 360                 }
 361
 362
 363         }
 364
 365         private void handleAreas(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 366                 List<PostfixTerm> areas = row.getLeveledAreas();
 367
 368                 for (PostfixTerm lArea : areas){
 369                         String description = lArea.term;
 370                         String abbrev = lArea.term;
 371                         NamedAreaType type = null;
 372                         String key = lArea.postfix + "_" + lArea.term;
 373                         UUID areaUuid = state.getArea(key);
 374                         NamedAreaLevel level = state.getPostfixLevel(lArea.postfix);
 375
 376                         TermMatchMode matchMode = state.getConfig().getAreaMatchMode();
 377                         NamedArea area = getNamedArea(state, areaUuid, lArea.term, description, abbrev, type, level, null, matchMode);
 378                         facade.addCollectingArea(area);
 379                         if (areaUuid == null){
 380                                 state.putArea(key, area.getUuid());
 381                         }
 382                 }
 383         }
 384
 385
 386         /**
 387          * @param state
 388          * @param row
 389          * @param facade
 390          */
 391         private void handleDeterminations(SpecimenCdmExcelImportState state,SpecimenRow row, DerivedUnitFacade facade) {
 392                 boolean isFirstDetermination = true;
 393                 DeterminationLight commonDetermination = row.getCommonDetermination();
 394                 Taxon commonTaxon = null;
 395                 TaxonNameBase<?,?> commonName = null;
 396
 397                 boolean hasCommonTaxonInfo = (commonDetermination == null) ? false : commonDetermination.hasTaxonInformation();
 398                 if (hasCommonTaxonInfo && commonDetermination != null){
 399                         TaxonBase<?> taxonBase = null;
 400                         if (StringUtils.isNotBlank(commonDetermination.taxonUuid)){
 401                                 UUID taxonUuid = UUID.fromString(commonDetermination.taxonUuid);
 402                                 taxonBase = getTaxonService().find(taxonUuid);
 403                                 if (taxonBase == null){
 404                                         String message = "Taxon for uuid %s not found in line %d.";
 405                                         message = String.format(message, taxonUuid.toString(), state.getCurrentLine());
 406                                         logger.warn(message);
 407                                 }
 408                         }else{
 409                                 taxonBase = findBestMatchingTaxon(state, commonDetermination, state.getConfig().isCreateTaxonIfNotExists());
 410                         }
 411                         commonTaxon = getAcceptedTaxon(taxonBase);
 412                         if (taxonBase != null){
 413                                 commonName = taxonBase.getName();
 414                         }else{
 415                                 commonTaxon = createTaxonFromDetermination(state, commonDetermination);
 416                                 commonName = commonTaxon.getName();
 417                         }
 418                 }
 419
 420
 421                 for (DeterminationLight determinationLight : row.getDetermination()){
 422                         Taxon taxon;
 423                         if (! hasCommonTaxonInfo){
 424                                 taxon = findBestMatchingTaxon(state, determinationLight, state.getConfig().isCreateTaxonIfNotExists());
 425                         }else{
 426                                 taxon = commonTaxon;
 427                         }
 428                         if (taxon != null){
 429                                 getTaxonService().saveOrUpdate(taxon);
 430                                 if (state.getConfig().isMakeIndividualAssociations() && taxon != null){
 431                                         IndividualsAssociation indivAssociciation = IndividualsAssociation.NewInstance();
 432                                         DerivedUnit du = facade.innerDerivedUnit();
 433                                         indivAssociciation.setAssociatedSpecimenOrObservation(du);
 434                                         getTaxonDescription(taxon).addElement(indivAssociciation);
 435                                         Feature feature = Feature.INDIVIDUALS_ASSOCIATION();
 436                                         if (facade.getType().isPreservedSpecimen()){
 437                                                 feature = Feature.SPECIMEN();
 438                                         }else if (facade.getType().isFeatureObservation()){
 439                                                 feature = Feature.OBSERVATION();
 440                                         }
 441                                         if (state.getConfig().isUseMaterialsExaminedForIndividualsAssociations()){
 442                                                 feature = Feature.MATERIALS_EXAMINED();
 443                                         }
 444
 445                                         indivAssociciation.setFeature(feature);
 446                                 }
 447                                 if (state.getConfig().isDeterminationsAreDeterminationEvent()){
 448                                         DeterminationEvent detEvent = makeDeterminationEvent(state, determinationLight, taxon);
 449                                         detEvent.setPreferredFlag(isFirstDetermination);
 450                                         facade.addDetermination(detEvent);
 451                                 }
 452                         }
 453
 454                         if (isFirstDetermination && state.getConfig().isFirstDeterminationIsStoredUnder()){
 455                                 TaxonNameBase<?,?> name;
 456
 457                                 if (!hasCommonTaxonInfo){
 458                                         name = findBestMatchingName(state, determinationLight);
 459                                 }else{
 460                                         if (commonName == null){
 461                                                 commonName = findBestMatchingName(state, commonDetermination);
 462                                         }
 463                                         name = commonName;
 464                                 }
 465                                 if (name != null){
 466                                         facade.setStoredUnder(name);
 467                                 }
 468                         }
 469                         isFirstDetermination = false;
 470                 }
 471         }
 472
 473         private Taxon createTaxonFromDetermination( SpecimenCdmExcelImportState state, DeterminationLight commonDetermination) {
 474
 475                 //rank
 476                 Rank rank;
 477                 try {
 478                         rank = StringUtils.isBlank(commonDetermination.rank) ? null : Rank.getRankByNameOrIdInVoc(commonDetermination.rank, true);
 479                 } catch (UnknownCdmTypeException e) {
 480                         rank = null;
 481                 }
 482
 483                 //name
 484                 NonViralName<?> name;
 485                 INonViralNameParser<NonViralName> parser = NonViralNameParserImpl.NewInstance();
 486                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 487                 if (StringUtils.isNotBlank(commonDetermination.fullName)){
 488                         name = parser.parseFullName(commonDetermination.fullName, nc, rank);
 489                         if (StringUtils.isBlank(name.getAuthorshipCache()) && StringUtils.isNotBlank(commonDetermination.author)){
 490                                 setAuthorship(name, commonDetermination.author, parser);
 491                         }
 492                 }else{
 493                         if (nc != null){
 494                                 name = (NonViralName)nc.getNewTaxonNameInstance(rank);
 495                         }else{
 496                                 name = NonViralName.NewInstance(rank);
 497                         }
 498                         if (StringUtils.isNotBlank(commonDetermination.genus)){
 499                                 name.setGenusOrUninomial(commonDetermination.genus);
 500                         }
 501                         if (StringUtils.isNotBlank(commonDetermination.speciesEpi)){
 502                                 name.setSpecificEpithet(commonDetermination.speciesEpi);
 503                         }
 504                         if (StringUtils.isNotBlank(commonDetermination.infraSpeciesEpi)){
 505                                 name.setInfraSpecificEpithet(commonDetermination.infraSpeciesEpi);
 506                         }
 507                         if (StringUtils.isNotBlank(commonDetermination.author)){
 508                                 setAuthorship(name, commonDetermination.author, parser);
 509                         }
 510                         //guess rank if null
 511                         if (name.getRank() == null){
 512                                 if (name.getInfraGenericEpithet() != null && name.getSpecificEpithet() == null){
 513                                         name.setRank(Rank.INFRAGENERICTAXON());
 514                                 }else if (name.getSpecificEpithet() != null && name.getInfraSpecificEpithet() == null){
 515                                         name.setRank(Rank.SPECIES());
 516                                 }else if (name.getInfraSpecificEpithet() != null){
 517                                         name.setRank(Rank.INFRASPECIFICTAXON());
 518                                 }
 519
 520                         }
 521
 522                 }
 523                 //sec
 524                 Reference<?> sec = null;
 525                 if (StringUtils.isNotBlank(commonDetermination.determinedBy)){
 526                         sec = ReferenceFactory.newGeneric();
 527                         TeamOrPersonBase<?> determinedBy;
 528                         BotanicalName dummyName = BotanicalName.NewInstance(Rank.SPECIES());
 529                         try {
 530                                 parser.parseAuthors(dummyName, commonDetermination.determinedBy);
 531                                 determinedBy = (TeamOrPersonBase<?>)dummyName.getCombinationAuthorTeam();
 532                         } catch (StringNotParsableException e) {
 533                                 determinedBy = Team.NewTitledInstance(commonDetermination.determinedBy, commonDetermination.determinedBy);
 534                         }
 535                         sec.setAuthorTeam(determinedBy);
 536                 }
 537
 538                 //taxon
 539                 Taxon taxon = Taxon.NewInstance(name, sec);
 540
 541                 if (StringUtils.isNotBlank(commonDetermination.family)){
 542                         if (name.getRank() == null || name.getRank().isLower(Rank.FAMILY()) ){
 543                                 logger.warn("Family taxon could not be created");
 544                         }
 545                 }
 546
 547                 //return
 548                 return taxon;
 549
 550         }
 551
 552
 553
 554
 555         private void setAuthorship(NonViralName<?> name, String author, INonViralNameParser<NonViralName> parser) {
 556                 if (name.isInstanceOf(BotanicalName.class) || name.isInstanceOf(ZoologicalName.class)){
 557                         try {
 558                                 parser.parseAuthors(name, author);
 559                         } catch (StringNotParsableException e) {
 560                                 name.setAuthorshipCache(author);
 561                         }
 562                 }else{
 563                         name.setAuthorshipCache(author);
 564                 }
 565
 566         }
 567
 568
 569
 570
 571         /**
 572          * This method tries to find the best matching taxon depending on the import configuration,
 573          * the taxon name information and the concept information available.
 574          *
 575          *
 576          * @param state
 577          * @param determinationLight
 578          * @param createIfNotExists
 579          * @return
 580          */
 581         private Taxon findBestMatchingTaxon(SpecimenCdmExcelImportState state, DeterminationLight determinationLight, boolean createIfNotExists) {
 582                 NonViralName<?> name = makeTaxonName(state, determinationLight);
 583
 584                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 585
 586                 if (! StringUtils.isBlank(titleCache)){
 587                         MatchingTaxonConfigurator matchConfigurator = MatchingTaxonConfigurator.NewInstance();
 588                         matchConfigurator.setTaxonNameTitle(titleCache);
 589                         matchConfigurator.setIncludeSynonyms(false);
 590                         Taxon taxon = getTaxonService().findBestMatchingTaxon(matchConfigurator);
 591
 592                         if(taxon == null && createIfNotExists){
 593                                 logger.info("creating new Taxon from TaxonName '" + titleCache+"'");
 594                                 UUID secUuid = null; //TODO
 595                                 Reference<?> sec = null;
 596                                 if (secUuid != null){
 597                                         sec = getReferenceService().find(secUuid);
 598                                 }
 599                                 taxon = Taxon.NewInstance(name, sec);
 600                         }else if (taxon == null){
 601                                 String message = "Taxon '%s' not found in line %d";
 602                                 message = String.format(message, titleCache, state.getCurrentLine());
 603                                 logger.warn(message);
 604                         }
 605                         return taxon;
 606                 }else {
 607                         return null;
 608                 }
 609         }
 610
 611         /**
 612          * @param state
 613          * @param determinationLight
 614          * @param name
 615          * @return
 616          */
 617         private String makeSearchNameTitleCache(SpecimenCdmExcelImportState state, DeterminationLight determinationLight,
 618                                 NonViralName<?> name) {
 619                 String titleCache = determinationLight.fullName;
 620                 if (! state.getConfig().isPreferNameCache() || StringUtils.isBlank(titleCache) ){
 621                         String computedTitleCache = name.getTitleCache();
 622                         if (StringUtils.isNotBlank(computedTitleCache)){
 623                                 titleCache = computedTitleCache;
 624                         }
 625
 626                 }
 627                 return titleCache;
 628         }
 629
 630         /**
 631          * @param state
 632          * @param determinationLight
 633          * @return
 634          */
 635         private NonViralName<?> makeTaxonName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 636                 NonViralName<?> name = NonViralName.NewInstance(null);
 637                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 638                 if (nc != null){
 639                         name = (NonViralName<?>)nc.getNewTaxonNameInstance(null);
 640                 }
 641                 name.setGenusOrUninomial(determinationLight.genus);
 642                 name.setSpecificEpithet(determinationLight.speciesEpi);
 643                 name.setInfraSpecificEpithet(determinationLight.infraSpeciesEpi);
 644
 645                 //FIXME bracketAuthors and teams not yet implemented!!!
 646                 List<String> authors = new ArrayList<String>();
 647                 if (StringUtils.isNotBlank(determinationLight.author)){
 648                         authors.add(determinationLight.author);
 649                 }
 650                 TeamOrPersonBase<?> agent = (TeamOrPersonBase)getOrMakeAgent(state, authors);
 651                 name.setCombinationAuthorTeam(agent);
 652
 653                 try {
 654                         if (StringUtils.isNotBlank(determinationLight.rank) ){
 655                                 name.setRank(Rank.getRankByNameOrIdInVoc(determinationLight.rank, nc, true));
 656                         }
 657                 } catch (UnknownCdmTypeException e) {
 658                         String message = "Rank not found: %s: ";
 659                         message = String.format(message, determinationLight.rank);
 660                         logger.warn(message);
 661                 }
 662                 if (StringUtils.isBlank(name.getInfraSpecificEpithet()) && StringUtils.isNotBlank(name.getSpecificEpithet() )){
 663                         name.setRank(Rank.SPECIES());
 664                 }
 665                 if (StringUtils.isBlank(name.getSpecificEpithet()) && StringUtils.isNotBlank(name.getGenusOrUninomial() )){
 666                         name.setRank(Rank.SPECIES());
 667                 }
 668                 if (StringUtils.isBlank(name.getTitleCache())){
 669                         //TODO test
 670                         name.setTitleCache(determinationLight.fullName, true);
 671                 }
 672                 return name;
 673         }
 674
 675         private TaxonNameBase findBestMatchingName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 676
 677                 NonViralName<?> name = makeTaxonName(state, determinationLight);
 678                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 679
 680                 //TODO
 681                 List<TaxonNameBase> matchingNames = getNameService().findByName(null, titleCache, MatchMode.EXACT, null, null, null, null, null).getRecords();
 682                 if (matchingNames.size() > 0){
 683                         return matchingNames.get(0);
 684                 } else if (matchingNames.size() > 0){
 685                         logger.warn("Get best matching taxon name not yet fully implemeted for specimen import");
 686                         return matchingNames.get(0);
 687                 }else{
 688                         return null;
 689                 }
 690
 691         }
 692
 693
 694         private DeterminationEvent makeDeterminationEvent(SpecimenCdmExcelImportState state, DeterminationLight determination, Taxon taxon) {
 695                 DeterminationEvent event = DeterminationEvent.NewInstance();
 696                 //taxon
 697                 event.setTaxon(taxon);
 698
 699                 //date
 700                 TimePeriod date = TimePeriodParser.parseString(determination.determinedWhen);
 701                 event.setTimeperiod(date);
 702                 //by
 703                 //FIXME bracketAuthors and teams not yet implemented!!!
 704                 List<String> authors = new ArrayList<String>();
 705                 if (StringUtils.isNotBlank(determination.determinedBy)){
 706                         authors.add(determination.determinedBy);
 707                 }
 708                 TeamOrPersonBase<?> actor = getOrMakeAgent(state, authors);
 709                 TeamOrPersonBase<?> secAuthor = taxon.getSec() == null ? null : taxon.getSec().getAuthorTeam();
 710                 if (actor != null && secAuthor != null && secAuthor.getTitleCache().equals(actor.getTitleCache()) && secAuthor.getNomenclaturalTitle().equals(actor.getNomenclaturalTitle())) {
 711                         actor = secAuthor;
 712                 }
 713
 714                 event.setActor(actor);
 715
 716                 //TODO
 717                 if (StringUtils.isNotBlank(determination.modifier)){
 718                         logger.warn("DeterminationModifiers not yet implemented for specimen import");
 719                 }
 720 //              DeterminationModifier modifier = DeterminationModifier.NewInstance(term, label, labelAbbrev);
 721 //              determination.modifier;
 722                 //notes
 723                 if (StringUtils.isNotEmpty(determination.notes)){
 724                         Annotation annotation = Annotation.NewInstance(determination.notes, AnnotationType.EDITORIAL(), Language.DEFAULT());
 725                         event.addAnnotation(annotation);
 726                 }
 727
 728                 return event;
 729         }
 730
 731         private TaxonDescription getTaxonDescription(Taxon taxon) {
 732                 TaxonDescription desc = this.getTaxonDescription(taxon, ! IMAGE_GALLERY, CREATE);
 733                 return desc;
 734         }
 735
 736         private TeamOrPersonBase<?> getOrMakeAgent(SpecimenCdmExcelImportState state, List<String> agents) {
 737                 if (agents.size() == 0){
 738                         return null;
 739                 }else if (agents.size() == 1){
 740                         return getOrMakePerson(state, agents.get(0));
 741                 }else{
 742                         return getOrMakeTeam(state, agents);
 743                 }
 744         }
 745
 746         private Person getOrMakePrimaryCollector(DerivedUnitFacade facade, String primaryCollector, SpecimenCdmExcelImportState state) {
 747                 if (StringUtils.isBlank(primaryCollector)){
 748                         return null;
 749                 }
 750                 AgentBase<?> collector = facade.getCollector();
 751                 List<Person> collectors = new ArrayList<Person>();
 752                 if (collector.isInstanceOf(Team.class) ){
 753                         Team team = CdmBase.deproxy(collector, Team.class);
 754                         collectors.addAll(team.getTeamMembers());
 755                 }else if (collector.isInstanceOf(Person.class)){
 756                         collectors.add(CdmBase.deproxy(collector, Person.class));
 757                 }else{
 758                         throw new IllegalStateException("Unknown subclass of agentbase: " + collector.getClass().getName() );
 759                 }
 760                 for (Person person :collectors){
 761                         if (primaryCollector.equalsIgnoreCase(person.getTitleCache())){
 762                                 return person;
 763                         }
 764                         if (primaryCollector.equalsIgnoreCase(person.getNomenclaturalTitle())){
 765                                 return person;
 766                         }
 767                 }
 768                 String message = "Primary Agent '%s' could not be determined in collector(s) in line %d";
 769                 message = String.format(message, primaryCollector, state.getCurrentLine());
 770                 logger.warn(message);
 771                 return null;
 772         }
 773
 774         private Team getOrMakeTeam(SpecimenCdmExcelImportState state, List<String> agents) {
 775                 String key = CdmUtils.concat("_", agents.toArray(new String[0]));
 776
 777                 Team result = state.getTeam(key);
 778                 if (result == null){
 779                         result = Team.NewInstance();
 780                         for (String member : agents){
 781                                 Person person = getOrMakePerson(state, member);
 782                                 result.addTeamMember(person);
 783                         }
 784                         state.putTeam(key, result);
 785                 }
 786                 return result;
 787         }
 788
 789         private Person getOrMakePerson(SpecimenCdmExcelImportState state, String value) {
 790                 Person result = state.getPerson(value);
 791                 if (result == null){
 792                         result = Person.NewInstance();
 793                         result.setTitleCache(value, true);
 794                         state.putPerson(value, result);
 795                 }
 796                 return result;
 797         }
 798
 799         private Reference<?> getOrMakeReference(SpecimenCdmExcelImportState state, String value) {
 800                 Reference<?> result = state.getReference(value);
 801                 if (result == null){
 802                         result = ReferenceFactory.newGeneric();
 803                         result.setTitleCache(value, true);
 804                         state.putReference(value, result);
 805                 }
 806                 return result;
 807         }
 808
 809
 810
 811         private Collection getOrMakeCollection(SpecimenCdmExcelImportState state, String collectionCode, String collectionString) {
 812                 Collection result = state.getCollection(collectionCode);
 813                 if (result == null){
 814                         result = Collection.NewInstance();
 815                         result.setCode(collectionCode);
 816                         result.setName(collectionString);
 817                         state.putCollection(collectionCode, result);
 818                 }
 819                 return result;
 820         }
 821
 822
 823         private TaxonNameBase<?, ?> getTaxonName(SpecimenCdmExcelImportState state, String name) {
 824                 TaxonNameBase<?,?> result = null;
 825                 result = state.getName(name);
 826                 if (result != null){
 827                         return result;
 828                 }
 829                 List<TaxonNameBase<?,?>> list = getNameService().findNamesByTitle(name);
 830                 //TODO better strategy to find best name, e.g. depending on the classification it is used in
 831                 if (! list.isEmpty()){
 832                         result = list.get(0);
 833                 }
 834                 if (result == null){
 835                         NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
 836                         NomenclaturalCode code = state.getConfig().getNomenclaturalCode();
 837                         result = parser.parseFullName(name, code, null);
 838
 839                 }
 840                 if (result != null){
 841                         state.putName(name, result);
 842                 }
 843                 return result;
 844         }
 845
 846         private SpecimenTypeDesignationStatus getSpecimenTypeStatus(SpecimenCdmExcelImportState state, String key)  {
 847                 SpecimenTypeDesignationStatus result = null;
 848                 try {
 849                         result = state.getTransformer().getSpecimenTypeDesignationStatusByKey(key);
 850                         if (result == null){
 851                                 String message = "Type status not recognized for %s in line %d";
 852                                 message = String.format(message, key, state.getCurrentLine());
 853                                 logger.warn(message);
 854                         }
 855                         return result;
 856                 } catch (UndefinedTransformerMethodException e) {
 857                         throw new RuntimeException("getSpecimenTypeDesignationStatusByKey not yet implemented");
 858                 }
 859
 860
 861         }
 862
 863
 864         private void handleExactLocation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 865
 866                 //reference system
 867                 ReferenceSystem refSys = null;
 868                 if (StringUtils.isNotBlank(row.getReferenceSystem())){
 869                         String strRefSys = row.getReferenceSystem().trim().replaceAll("\\s", "");
 870                         UUID refUuid;
 871                         try {
 872                                 refSys = state.getTransformer().getReferenceSystemByKey(strRefSys);
 873                                 if (refSys == null){
 874                                         refUuid = state.getTransformer().getReferenceSystemUuid(strRefSys);
 875                                         if (refUuid == null){
 876                                                 String message = "Unknown reference system %s in line %d";
 877                                                 message = String.format(message, strRefSys, state.getCurrentLine());
 878                                                 logger.warn(message);
 879                                         }
 880                                         refSys = getReferenceSystem(state, refUuid, strRefSys, strRefSys, strRefSys, null);
 881                                 }
 882
 883                         } catch (UndefinedTransformerMethodException e) {
 884                                 throw new RuntimeException(e);
 885                         }
 886                 }
 887
 888
 889
 890                 // lat/ long /error
 891                 try {
 892                         String longitude = row.getLongitude();
 893                         String latitude = row.getLatitude();
 894                         Integer errorRadius = null;
 895                         if (StringUtils.isNotBlank(row.getErrorRadius())){
 896                                 try {
 897                                         errorRadius = Integer.valueOf(row.getErrorRadius());
 898                                 } catch (NumberFormatException e) {
 899                                         String message = "Error radius %s could not be transformed to Integer in line %d";
 900                                         message = String.format(message, row.getErrorRadius(), state.getCurrentLine());
 901                                         logger.warn(message);
 902                                 }
 903                         }
 904                         //all
 905                         facade.setExactLocationByParsing(longitude, latitude, refSys, errorRadius);
 906                 } catch (ParseException e) {
 907                         String message = "Problems when parsing exact location for line %d";
 908                         message = String.format(message, state.getCurrentLine());
 909                         logger.warn(message);
 910
 911                 }
 912
 913
 914
 915         }
 916
 917
 918         /*
 919          * Set the current Country
 920          * Search in the DB if the isoCode is known
 921          * If not, search if the country name is in the DB
 922          * If not, create a new Label with the Level Country
 923          * @param iso: the country iso code
 924          * @param fullName: the country's full name
 925          * @param app: the CDM application controller
 926          */
 927         private void handleCountry(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 928
 929                 if (StringUtils.isNotBlank(row.getIsoCountry())){
 930                         NamedArea country = getOccurrenceService().getCountryByIso(row.getIsoCountry());
 931                         if (country != null){
 932                                 facade.setCountry(country);
 933                                 return;
 934                         }
 935                 }
 936                 if (StringUtils.isNotBlank(row.getCountry())){
 937                         List<Country> countries = getOccurrenceService().getCountryByName(row.getCountry());
 938                         if (countries.size() >0){
 939                                 facade.setCountry(countries.get(0));
 940                         }else{
 941                                 UUID uuid = UUID.randomUUID();
 942                                 String label = row.getCountry();
 943                                 String text = row.getCountry();
 944                                 String labelAbbrev = null;
 945                                 NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
 946                                 NamedAreaLevel level = NamedAreaLevel.COUNTRY();
 947                                 NamedArea newCountry = this.getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level);
 948                                 facade.setCountry(newCountry);
 949                         }
 950                 }
 951         }
 952
 953         @Override
 954         protected void secondPass(SpecimenCdmExcelImportState state) {
 955                 //no second path defined yet
 956                 return;
 957         }
 958
 959
 960         @Override
 961         protected String getWorksheetName() {
 962                 return WORKSHEET_NAME;
 963         }
 964
 965         @Override
 966         protected boolean needsNomenclaturalCode() {
 967                 return false;
 968         }
 969
 970
 971         /* (non-Javadoc)
 972          * @see eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase#createDataHolderRow()
 973          */
 974         @Override
 975         protected SpecimenRow createDataHolderRow() {
 976                 return new SpecimenRow();
 977         }
 978
 979
 980
 981
 982         /* (non-Javadoc)
 983          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 984          */
 985         @Override
 986         protected boolean doCheck(SpecimenCdmExcelImportState state) {
 987                 logger.warn("Validation not yet implemented for " + this.getClass().getSimpleName());
 988                 return true;
 989         }
 990
 991
 992
 993         @Override
 994         protected boolean isIgnore(SpecimenCdmExcelImportState state) {
 995                 return !state.getConfig().isDoSpecimen();
 996         }
 997
 998
 999 }