cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenCdmExcelImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen.excel.in;
  11
  12 import java.text.ParseException;
  13 import java.util.ArrayList;
  14 import java.util.List;
  15 import java.util.UUID;
  16
  17 import org.apache.commons.lang.StringUtils;
  18 import org.apache.log4j.Logger;
  19 import org.springframework.stereotype.Component;
  20
  21 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  22 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  23 import eu.etaxonomy.cdm.api.service.config.MatchingTaxonConfigurator;
  24 import eu.etaxonomy.cdm.common.CdmUtils;
  25 import eu.etaxonomy.cdm.io.common.ICdmIO;
  26 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  27 import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase.PostfixTerm;
  28 import eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase;
  29 import eu.etaxonomy.cdm.io.specimen.excel.in.SpecimenRow.DeterminationLight;
  30 import eu.etaxonomy.cdm.model.agent.AgentBase;
  31 import eu.etaxonomy.cdm.model.agent.Person;
  32 import eu.etaxonomy.cdm.model.agent.Team;
  33 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  34 import eu.etaxonomy.cdm.model.common.Annotation;
  35 import eu.etaxonomy.cdm.model.common.AnnotationType;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.Language;
  39 import eu.etaxonomy.cdm.model.common.TimePeriod;
  40 import eu.etaxonomy.cdm.model.description.Feature;
  41 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  42 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  43 import eu.etaxonomy.cdm.model.location.NamedArea;
  44 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  45 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  46 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  47 import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
  48 import eu.etaxonomy.cdm.model.name.BotanicalName;
  49 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  50 import eu.etaxonomy.cdm.model.name.NonViralName;
  51 import eu.etaxonomy.cdm.model.name.Rank;
  52 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  53 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  54 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  55 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  56 import eu.etaxonomy.cdm.model.occurrence.Collection;
  57 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
  58 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  59 import eu.etaxonomy.cdm.model.reference.Reference;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  61 import eu.etaxonomy.cdm.model.taxon.Taxon;
  62 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  63 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  64 import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
  65 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  66 import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
  67 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  68
  69 /**
  70  * @author a.mueller
  71  * @created 10.05.2011
  72  * @version 1.0
  73  */
  74 @Component
  75 public class SpecimenCdmExcelImport  extends ExcelTaxonOrSpecimenImportBase<SpecimenCdmExcelImportState, SpecimenRow>  implements ICdmIO<SpecimenCdmExcelImportState> {
  76         private static final Logger logger = Logger.getLogger(SpecimenCdmExcelImport.class);
  77
  78         private static final String WORKSHEET_NAME = "Specimen";
  79
  80         private static final String BASIS_OF_RECORD_COLUMN = "(?i)(BasisOfRecord)";
  81         private static final String COUNTRY_COLUMN = "(?i)(Country)";
  82         private static final String AREA_COLUMN = "(?i)(Area)";
  83         private static final String ISO_COUNTRY_COLUMN = "(?i)(ISOCountry|CountryCode)";
  84         private static final String LOCALITY_COLUMN = "(?i)(Locality)";
  85         private static final String ALTITUDE_COLUMN = "(?i)(AbsoluteElevation|Altitude)";
  86         private static final String ALTITUDE_MAX_COLUMN = "(?i)(AbsoluteElevation|Altitude)Max(imum)?";
  87         private static final String COLLECTION_DATE_COLUMN = "(?i)(CollectionDate)";
  88         private static final String COLLECTION_DATE_END_COLUMN = "(?i)(CollectionDateEnd)";
  89         private static final String COLLECTOR_COLUMN = "(?i)(Collector)";
  90         private static final String COLLECTORS_COLUMN = "(?i)(Collectors)";
  91         private static final String PRIMARY_COLLECTOR_COLUMN = "(?i)(PrimaryCollector)";
  92         private static final String LONGITUDE_COLUMN = "(?i)(Longitude)";
  93         private static final String LATITUDE_COLUMN = "(?i)(Latitude)";
  94         private static final String REFERENCE_SYSTEM_COLUMN = "(?i)(ReferenceSystem)";
  95         private static final String ERROR_RADIUS_COLUMN = "(?i)(ErrorRadius)";
  96
  97
  98         private static final String COLLECTORS_NUMBER_COLUMN = "(?i)((Collectors|Field)Number)";
  99         private static final String ECOLOGY_COLUMN = "(?i)(Ecology|Habitat)";
 100         private static final String PLANT_DESCRIPTION_COLUMN = "(?i)(PlantDescription)";
 101         private static final String FIELD_NOTES_COLUMN = "(?i)(FieldNotes)";
 102         private static final String SEX_COLUMN = "(?i)(Sex)";
 103
 104
 105         private static final String ACCESSION_NUMBER_COLUMN = "(?i)(AccessionNumber)";
 106         private static final String BARCODE_COLUMN = "(?i)(Barcode)";
 107         private static final String COLLECTION_CODE_COLUMN = "(?i)(CollectionCode)";
 108         private static final String COLLECTION_COLUMN = "(?i)(Collection)";
 109         private static final String UNIT_NOTES_COLUMN = "(?i)((Unit)?Notes)";
 110
 111
 112         private static final String TYPE_CATEGORY_COLUMN = "(?i)(TypeCategory)";
 113         private static final String TYPIFIED_NAME_COLUMN = "(?i)(TypifiedName|TypeOf)";
 114
 115
 116         private static final String SOURCE_COLUMN = "(?i)(Source)";
 117         private static final String ID_IN_SOURCE_COLUMN = "(?i)(IdInSource)";
 118
 119
 120         private static final String DETERMINATION_AUTHOR_COLUMN = "(?i)(Author)";
 121         private static final String DETERMINATION_MODIFIER_COLUMN = "(?i)(DeterminationModifier)";
 122         private static final String DETERMINED_BY_COLUMN = "(?i)(DeterminationBy)";
 123         private static final String DETERMINED_WHEN_COLUMN = "(?i)(DeterminationWhen)";
 124         private static final String DETERMINATION_NOTES_COLUMN = "(?i)(DeterminationNote)";
 125         private static final String EXTENSION_COLUMN = "(?i)(Ext(ension)?)";
 126
 127
 128         public SpecimenCdmExcelImport() {
 129                 super();
 130         }
 131
 132
 133
 134
 135         @Override
 136         protected void analyzeSingleValue(KeyValue keyValue, SpecimenCdmExcelImportState state) {
 137                 SpecimenRow row = state.getCurrentRow();
 138                 String value = keyValue.value;
 139                 if(keyValue.key.matches(BASIS_OF_RECORD_COLUMN)) {
 140                         row.setBasisOfRecord(value);
 141                 } else if(keyValue.key.matches(COUNTRY_COLUMN)) {
 142                         row.setCountry(value);
 143                 } else if(keyValue.key.matches(ISO_COUNTRY_COLUMN)) {
 144                         row.setIsoCountry(value);
 145                 } else if(keyValue.key.matches(LOCALITY_COLUMN)) {
 146                         row.setLocality(value);
 147                 } else if(keyValue.key.matches(FIELD_NOTES_COLUMN)) {
 148                         row.setLocality(value);
 149                 } else if(keyValue.key.matches(ALTITUDE_COLUMN)) {
 150                         row.setAltitude(value);
 151                 } else if(keyValue.key.matches(ALTITUDE_MAX_COLUMN)) {
 152                         row.setAltitudeMax(value);
 153                 } else if(keyValue.key.matches(COLLECTOR_COLUMN)) {
 154                         row.putCollector(keyValue.index, value);
 155                 } else if(keyValue.key.matches(PRIMARY_COLLECTOR_COLUMN)) {
 156                         row.setPrimaryCollector(value);
 157                 } else if(keyValue.key.matches(ECOLOGY_COLUMN)) {
 158                         row.setEcology(value);
 159                 } else if(keyValue.key.matches(PLANT_DESCRIPTION_COLUMN)) {
 160                         row.setPlantDescription(value);
 161                 } else if(keyValue.key.matches(SEX_COLUMN)) {
 162                         row.setSex(value);
 163                 } else if(keyValue.key.matches(COLLECTION_DATE_COLUMN)) {
 164                         row.setCollectingDate(value);
 165                 } else if(keyValue.key.matches(COLLECTION_DATE_END_COLUMN)) {
 166                         row.setCollectingDateEnd(value);
 167                 } else if(keyValue.key.matches(COLLECTORS_COLUMN)) {
 168                         row.setCollectors(value);
 169                 } else if(keyValue.key.matches(COLLECTOR_COLUMN)) {
 170                         row.putCollector(keyValue.index, value);
 171                 } else if(keyValue.key.matches(COLLECTORS_NUMBER_COLUMN)) {
 172                         row.setCollectorsNumber(value);
 173                 } else if(keyValue.key.matches(LONGITUDE_COLUMN)) {
 174                         row.setLongitude(value);
 175                 } else if(keyValue.key.matches(LATITUDE_COLUMN)) {
 176                         row.setLatitude(value);
 177                 } else if(keyValue.key.matches(REFERENCE_SYSTEM_COLUMN)) {
 178                         row.setReferenceSystem(value);
 179                 } else if(keyValue.key.matches(ERROR_RADIUS_COLUMN)) {
 180                         row.setErrorRadius(value);
 181                 } else if(keyValue.key.matches(AREA_COLUMN)) {
 182                         if (keyValue.postfix != null){
 183                                 row.addLeveledArea(keyValue.postfix, value);
 184                         }else{
 185                                 logger.warn("Not yet implemented");
 186                         }
 187
 188
 189
 190                 } else if(keyValue.key.matches(ACCESSION_NUMBER_COLUMN)) {
 191                         row.setLocality(value);
 192                 } else if(keyValue.key.matches(BARCODE_COLUMN)) {
 193                         row.setBarcode(value);
 194                 } else if(keyValue.key.matches(UNIT_NOTES_COLUMN)) {
 195                         row.putUnitNote(keyValue.index, value);
 196
 197
 198                 } else if(keyValue.key.matches(FAMILY_COLUMN)) {
 199                         row.putDeterminationFamily(keyValue.index, value);
 200                 } else if(keyValue.key.matches(GENUS_COLUMN)) {
 201                         row.putDeterminationGenus(keyValue.index, value);
 202                 } else if(keyValue.key.matches(SPECIFIC_EPITHET_COLUMN)) {
 203                         row.putDeterminationSpeciesEpi(keyValue.index, value);
 204                 } else if(keyValue.key.matches(INFRASPECIFIC_EPITHET_COLUMN)) {
 205                         row.putDeterminationInfraSpeciesEpi(keyValue.index, value);
 206                 } else if(keyValue.key.matches(RANK_COLUMN)) {
 207                         row.putDeterminationRank(keyValue.index, value);
 208                 } else if(keyValue.key.matches(TAXON_UUID_COLUMN)) {
 209                         row.putDeterminationTaxonUuid(keyValue.index, value);
 210                 } else if(keyValue.key.matches(FULL_NAME_COLUMN)) {
 211                         row.putDeterminationFullName(keyValue.index, value);
 212                 } else if(keyValue.key.matches(DETERMINATION_AUTHOR_COLUMN)) {
 213                         row.putDeterminationAuthor(keyValue.index, value);
 214                 } else if(keyValue.key.matches(DETERMINATION_MODIFIER_COLUMN)) {
 215                         row.putDeterminationDeterminationModifier(keyValue.index, value);
 216                 } else if(keyValue.key.matches(DETERMINATION_NOTES_COLUMN)) {
 217                         row.putDeterminationDeterminationNotes(keyValue.index, value);
 218                 } else if(keyValue.key.matches(DETERMINED_BY_COLUMN)) {
 219                         row.putDeterminationDeterminedBy(keyValue.index, value);
 220                 } else if(keyValue.key.matches(DETERMINED_WHEN_COLUMN)) {
 221                         row.putDeterminationDeterminedWhen(keyValue.index, value);
 222
 223                 } else if(keyValue.key.matches(COLLECTION_CODE_COLUMN)) {
 224                         row.setCollectionCode(value);
 225                 } else if(keyValue.key.matches(COLLECTION_COLUMN)) {
 226                         row.setCollection(value);
 227
 228                 } else if(keyValue.key.matches(TYPE_CATEGORY_COLUMN)) {
 229                         row.putTypeCategory(keyValue.index, getSpecimenTypeStatus(state, value));
 230                 } else if(keyValue.key.matches(TYPIFIED_NAME_COLUMN)) {
 231                         row.putTypifiedName(keyValue.index, getTaxonName(state, value));
 232
 233
 234                 } else if(keyValue.key.matches(SOURCE_COLUMN)) {
 235                         row.putSourceReference(keyValue.index, getOrMakeReference(state, value) );
 236                 } else if(keyValue.key.matches(ID_IN_SOURCE_COLUMN)) {
 237                         row.putIdInSource(keyValue.index, value);
 238                 } else if(keyValue.key.matches(EXTENSION_COLUMN)) {
 239                         if (keyValue.postfix != null){
 240                                 row.addExtension(keyValue.postfix, value);
 241                         }else{
 242                                 logger.warn("Extension without postfix not yet implemented");
 243                         }
 244
 245                 }else {
 246                         state.setUnsuccessfull();
 247                         logger.error("Unexpected column header " + keyValue.originalKey);
 248                 }
 249
 250         return;
 251         }
 252
 253
 254         @Override
 255         protected void firstPass(SpecimenCdmExcelImportState state) {
 256                 SpecimenRow row = state.getCurrentRow();
 257
 258                 //basis of record
 259                 DerivedUnitType type = DerivedUnitType.valueOf2(row.getBasisOfRecord());
 260                 if (type == null){
 261                         String message = "%s is not a valid BasisOfRecord. 'Unknown' is used instead in line %d.";
 262                         message = String.format(message, row.getBasisOfRecord(), state.getCurrentLine());
 263                         logger.warn(message);
 264                         type = DerivedUnitType.DerivedUnit;
 265                 }
 266                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(type);
 267
 268                 //country
 269                 handleCountry(facade, row, state);
 270                 handleAreas(facade,row, state);
 271
 272                 facade.setGatheringPeriod(getTimePeriod(row.getCollectingDate(), row.getCollectingDateEnd()));
 273                 facade.setLocality(row.getLocality());
 274                 facade.setFieldNotes(row.getFieldNotes());
 275                 facade.setFieldNumber(NB(row.getCollectorsNumber()));
 276                 facade.setEcology(NB(row.getEcology()));
 277                 facade.setPlantDescription(NB(row.getPlantDescription()));
 278 //              facade.setSex(row.get)
 279                 handleExactLocation(facade, row, state);
 280                 facade.setCollector(getOrMakeAgent(state, row.getCollectors()));
 281                 facade.setPrimaryCollector(getOrMakePrimaryCollector(facade, row.getPrimaryCollector(), state));
 282                 handleAbsoluteElevation(facade, row, state);
 283
 284
 285                 //derivedUnit
 286                 facade.setBarcode(NB(row.getBarcode()));
 287                 facade.setAccessionNumber(NB(row.getAccessionNumber()));
 288                 facade.setCollection(getOrMakeCollection(state, row.getCollectionCode(), row.getCollection()));
 289                 for (IdentifiableSource source : row.getSources()){
 290                         facade.addSource(source);
 291                 }
 292                 for (SpecimenTypeDesignation designation : row.getTypeDesignations()){
 293                         //FIXME
 294 //                      facade.innerDerivedUnit().addSpecimenTypeDesignation(designation);
 295                 }
 296                 handleDeterminations(state, row, facade);
 297                 handleExtensions(facade.innerDerivedUnit(),row, state);
 298                 for (String note : row.getUnitNotes()){
 299                         Annotation annotation = Annotation.NewInstance(note, AnnotationType.EDITORIAL(), Language.DEFAULT());
 300                         facade.addAnnotation(annotation);
 301                 }
 302
 303                 //save
 304                 getOccurrenceService().save(facade.innerDerivedUnit());
 305                 return;
 306         }
 307
 308         private void handleAbsoluteElevation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 309                 //altitude
 310
 311                 try {
 312                         String altitude = row.getAltitude();
 313                         if (StringUtils.isBlank(altitude)){
 314                                 return;
 315                         }
 316 //                      if (altitude.endsWith(".0")){
 317 //                              altitude = altitude.substring(0, altitude.length() -2);
 318 //                      }
 319                         int value = Integer.valueOf(altitude);
 320                         facade.setAbsoluteElevation(value);
 321                 } catch (NumberFormatException e) {
 322                         String message = "Absolute elevation / altitude '%s' is not an integer number in line %d";
 323                         message = String.format(message, row.getAltitude(), state.getCurrentLine());
 324                         logger.warn(message);
 325                         return;
 326                 }
 327
 328                 //max
 329
 330                 try {
 331                         String max = row.getAltitudeMax();
 332                         if (StringUtils.isBlank(max)){
 333                                 return;
 334                         }
 335 //                      if (max.endsWith(".0")){
 336 //                              max = max.substring(0, max.length() -2);
 337 //                      }
 338                         int value = Integer.valueOf(max);
 339                         //TODO avoid unequal distance
 340                         int min = facade.getAbsoluteElevation();
 341                         if ( (value - min) % 2 == 1 ){
 342                                 String message = "Altitude min-max difference ist not equal. Max reduced by 1 in line %d";
 343                                 message = String.format(message, state.getCurrentLine());
 344                                 logger.warn(message);
 345                                 value--;
 346                         }
 347                         facade.setAbsoluteElevationRange(min, value);
 348                 } catch (NumberFormatException e) {
 349                         String message = "Absolute elevation / Altitude maximum '%s' is not an integer number in line %d";
 350                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 351                         logger.warn(message);
 352                         return;
 353                 }catch (Exception e){
 354                         String message = "Error occurred when trying to write Absolute elevation / Altitude maximum '%s' in line %d";
 355                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 356                         logger.warn(message);
 357                         return;
 358
 359                 }
 360
 361
 362         }
 363
 364         private void handleAreas(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 365                 List<PostfixTerm> areas = row.getLeveledAreas();
 366
 367                 for (PostfixTerm lArea : areas){
 368                         String description = lArea.term;
 369                         String abbrev = lArea.term;
 370                         NamedAreaType type = null;
 371                         String key = lArea.postfix + "_" + lArea.term;
 372                         UUID areaUuid = state.getArea(key);
 373                         NamedAreaLevel level = state.getPostfixLevel(lArea.postfix);
 374
 375                         TermMatchMode matchMode = state.getConfig().getAreaMatchMode();
 376                         NamedArea area = getNamedArea(state, areaUuid, lArea.term, description, abbrev, type, level, null, matchMode);
 377                         facade.addCollectingArea(area);
 378                         if (areaUuid == null){
 379                                 state.putArea(key, area.getUuid());
 380                         }
 381                 }
 382         }
 383
 384
 385         /**
 386          * @param state
 387          * @param row
 388          * @param facade
 389          */
 390         private void handleDeterminations(SpecimenCdmExcelImportState state,SpecimenRow row, DerivedUnitFacade facade) {
 391                 boolean isFirstDetermination = true;
 392                 DeterminationLight commonDetermination = row.getCommonDetermination();
 393                 Taxon commonTaxon = null;
 394                 TaxonNameBase<?,?> commonName = null;
 395
 396                 boolean hasCommonTaxonInfo = commonDetermination == null ? false : commonDetermination.hasTaxonInformation();
 397                 if (hasCommonTaxonInfo && commonDetermination != null){
 398                         TaxonBase<?> taxonBase = null;
 399                         if (StringUtils.isNotBlank(commonDetermination.taxonUuid)){
 400                                 UUID taxonUuid = UUID.fromString(commonDetermination.taxonUuid);
 401                                 taxonBase = getTaxonService().find(taxonUuid);
 402                                 if (taxonBase == null){
 403                                         String message = "Taxon for uuid %s not found in line %d.";
 404                                         message = String.format(message, taxonUuid.toString(), state.getCurrentLine());
 405                                         logger.warn(message);
 406                                 }
 407                         }else{
 408                                 taxonBase = findBestMatchingTaxon(state, commonDetermination, state.getConfig().isCreateTaxonIfNotExists());
 409                         }
 410                         commonTaxon = getAcceptedTaxon(taxonBase);
 411                         if (taxonBase != null){
 412                                 commonName = taxonBase.getName();
 413                         }else{
 414                                 commonTaxon = createTaxonFromDetermination(state, commonDetermination);
 415                                 commonName = commonTaxon.getName();
 416                         }
 417                 }
 418
 419
 420                 for (DeterminationLight determinationLight : row.getDetermination()){
 421                         Taxon taxon;
 422                         if (! hasCommonTaxonInfo){
 423                                 taxon = findBestMatchingTaxon(state, determinationLight, state.getConfig().isCreateTaxonIfNotExists());
 424                         }else{
 425                                 taxon = commonTaxon;
 426                         }
 427                         if (taxon != null){
 428                                 getTaxonService().saveOrUpdate(taxon);
 429                                 if (state.getConfig().isMakeIndividualAssociations() && taxon != null){
 430                                         IndividualsAssociation indivAssociciation = IndividualsAssociation.NewInstance();
 431                                         DerivedUnitBase<?> du = facade.innerDerivedUnit();
 432                                         indivAssociciation.setAssociatedSpecimenOrObservation(du);
 433                                         getTaxonDescription(taxon).addElement(indivAssociciation);
 434                                         Feature feature = Feature.INDIVIDUALS_ASSOCIATION();
 435                                         if (facade.getType().equals(DerivedUnitType.Specimen)){
 436                                                 feature = Feature.SPECIMEN();
 437                                         }else if (facade.getType().equals(DerivedUnitType.Observation)){
 438                                                 feature = Feature.OBSERVATION();
 439                                         }
 440                                         if (state.getConfig().isUseMaterialsExaminedForIndividualsAssociations()){
 441                                                 feature = Feature.MATERIALS_EXAMINED();
 442                                         }
 443
 444                                         indivAssociciation.setFeature(feature);
 445                                 }
 446                                 if (state.getConfig().isDeterminationsAreDeterminationEvent()){
 447                                         DeterminationEvent detEvent = makeDeterminationEvent(state, determinationLight, taxon);
 448                                         detEvent.setPreferredFlag(isFirstDetermination);
 449                                         facade.addDetermination(detEvent);
 450                                 }
 451                         }
 452
 453                         if (isFirstDetermination && state.getConfig().isFirstDeterminationIsStoredUnder()){
 454                                 TaxonNameBase<?,?> name;
 455
 456                                 if (!hasCommonTaxonInfo){
 457                                         name = findBestMatchingName(state, determinationLight);
 458                                 }else{
 459                                         if (commonName == null){
 460                                                 commonName = findBestMatchingName(state, commonDetermination);
 461                                         }
 462                                         name = commonName;
 463                                 }
 464                                 if (name != null){
 465                                         facade.setStoredUnder(name);
 466                                 }
 467                         }
 468                         isFirstDetermination = false;
 469                 }
 470         }
 471
 472         private Taxon createTaxonFromDetermination( SpecimenCdmExcelImportState state, DeterminationLight commonDetermination) {
 473
 474                 //rank
 475                 Rank rank;
 476                 try {
 477                         rank = StringUtils.isBlank(commonDetermination.rank) ? null : Rank.getRankByNameOrAbbreviation(commonDetermination.rank, true);
 478                 } catch (UnknownCdmTypeException e) {
 479                         rank = null;
 480                 }
 481
 482                 //name
 483                 NonViralName<?> name;
 484                 INonViralNameParser<NonViralName> parser = NonViralNameParserImpl.NewInstance();
 485                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 486                 if (StringUtils.isNotBlank(commonDetermination.fullName)){
 487                         name = parser.parseFullName(commonDetermination.fullName, nc, rank);
 488                         if (StringUtils.isBlank(name.getAuthorshipCache()) && StringUtils.isNotBlank(commonDetermination.author)){
 489                                 setAuthorship(name, commonDetermination.author, parser);
 490                         }
 491                 }else{
 492                         if (nc != null){
 493                                 name = (NonViralName)nc.getNewTaxonNameInstance(rank);
 494                         }else{
 495                                 name = NonViralName.NewInstance(rank);
 496                         }
 497                         if (StringUtils.isNotBlank(commonDetermination.genus)){
 498                                 name.setGenusOrUninomial(commonDetermination.genus);
 499                         }
 500                         if (StringUtils.isNotBlank(commonDetermination.speciesEpi)){
 501                                 name.setSpecificEpithet(commonDetermination.speciesEpi);
 502                         }
 503                         if (StringUtils.isNotBlank(commonDetermination.infraSpeciesEpi)){
 504                                 name.setInfraSpecificEpithet(commonDetermination.infraSpeciesEpi);
 505                         }
 506                         if (StringUtils.isNotBlank(commonDetermination.author)){
 507                                 setAuthorship(name, commonDetermination.author, parser);
 508                         }
 509                         //guess rank if null
 510                         if (name.getRank() == null){
 511                                 if (name.getInfraGenericEpithet() != null && name.getSpecificEpithet() == null){
 512                                         name.setRank(Rank.INFRAGENERICTAXON());
 513                                 }else if (name.getSpecificEpithet() != null && name.getInfraSpecificEpithet() == null){
 514                                         name.setRank(Rank.SPECIES());
 515                                 }else if (name.getInfraSpecificEpithet() != null){
 516                                         name.setRank(Rank.INFRASPECIFICTAXON());
 517                                 }
 518
 519                         }
 520
 521                 }
 522                 //sec
 523                 Reference<?> sec = null;
 524                 if (StringUtils.isNotBlank(commonDetermination.determinedBy)){
 525                         sec = ReferenceFactory.newGeneric();
 526                         TeamOrPersonBase determinedBy;
 527                         BotanicalName dummyName = BotanicalName.NewInstance(Rank.SPECIES());
 528                         try {
 529                                 parser.parseAuthors(dummyName, commonDetermination.determinedBy);
 530                                 determinedBy = (TeamOrPersonBase)dummyName.getCombinationAuthorTeam();
 531                         } catch (StringNotParsableException e) {
 532                                 determinedBy = Team.NewTitledInstance(commonDetermination.determinedBy, commonDetermination.determinedBy);
 533                         }
 534                         sec.setAuthorTeam(determinedBy);
 535                 }
 536
 537                 //taxon
 538                 Taxon taxon = Taxon.NewInstance(name, sec);
 539
 540                 if (StringUtils.isNotBlank(commonDetermination.family)){
 541                         if (name.getRank() == null || name.getRank().isLower(Rank.FAMILY()) ){
 542                                 logger.warn("Family taxon could not be created");
 543                         }
 544                 }
 545
 546                 //return
 547                 return taxon;
 548
 549         }
 550
 551
 552
 553
 554         private void setAuthorship(NonViralName name, String author, INonViralNameParser<NonViralName> parser) {
 555                 if (name.isInstanceOf(BotanicalName.class) || name.isInstanceOf(ZoologicalName.class)){
 556                         try {
 557                                 parser.parseAuthors(name, author);
 558                         } catch (StringNotParsableException e) {
 559                                 name.setAuthorshipCache(author);
 560                         }
 561                 }else{
 562                         name.setAuthorshipCache(author);
 563                 }
 564
 565         }
 566
 567
 568
 569
 570         /**
 571          * This method tries to find the best matching taxon depending on the import configuration,
 572          * the taxon name information and the concept information available.
 573          *
 574          *
 575          * @param state
 576          * @param determinationLight
 577          * @param createIfNotExists
 578          * @return
 579          */
 580         private Taxon findBestMatchingTaxon(SpecimenCdmExcelImportState state, DeterminationLight determinationLight, boolean createIfNotExists) {
 581                 NonViralName<?> name = makeTaxonName(state, determinationLight);
 582
 583                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 584
 585                 if (! StringUtils.isBlank(titleCache)){
 586                         MatchingTaxonConfigurator matchConfigurator = MatchingTaxonConfigurator.NewInstance();
 587                         matchConfigurator.setTaxonNameTitle(titleCache);
 588                         matchConfigurator.setIncludeSynonyms(false);
 589                         Taxon taxon = getTaxonService().findBestMatchingTaxon(matchConfigurator);
 590
 591                         if(taxon == null && createIfNotExists){
 592                                 logger.info("creating new Taxon from TaxonName '" + titleCache+"'");
 593                                 UUID secUuid = null; //TODO
 594                                 Reference<?> sec = null;
 595                                 if (secUuid != null){
 596                                         sec = getReferenceService().find(secUuid);
 597                                 }
 598                                 taxon = Taxon.NewInstance(name, sec);
 599                         }else if (taxon == null){
 600                                 String message = "Taxon '%s' not found in line %d";
 601                                 message = String.format(message, titleCache, state.getCurrentLine());
 602                                 logger.warn(message);
 603                         }
 604                         return taxon;
 605                 }else {
 606                         return null;
 607                 }
 608         }
 609
 610         /**
 611          * @param state
 612          * @param determinationLight
 613          * @param name
 614          * @return
 615          */
 616         private String makeSearchNameTitleCache(SpecimenCdmExcelImportState state, DeterminationLight determinationLight,
 617                                 NonViralName name) {
 618                 String titleCache = determinationLight.fullName;
 619                 if (! state.getConfig().isPreferNameCache() || StringUtils.isBlank(titleCache) ){
 620                         String computedTitleCache = name.getTitleCache();
 621                         if (StringUtils.isNotBlank(computedTitleCache)){
 622                                 titleCache = computedTitleCache;
 623                         }
 624
 625                 }
 626                 return titleCache;
 627         }
 628
 629         /**
 630          * @param state
 631          * @param determinationLight
 632          * @return
 633          */
 634         private NonViralName makeTaxonName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 635                 //TODO correct type by config.nc
 636                 NonViralName name =NonViralName.NewInstance(null);
 637                 name.setGenusOrUninomial(determinationLight.genus);
 638                 name.setSpecificEpithet(determinationLight.speciesEpi);
 639                 name.setInfraSpecificEpithet(determinationLight.infraSpeciesEpi);
 640
 641                 //FIXME bracketAuthors and teams not yet implemented!!!
 642                 List<String> authors = new ArrayList<String>();
 643                 if (StringUtils.isNotBlank(determinationLight.author)){
 644                         authors.add(determinationLight.author);
 645                 }
 646                 TeamOrPersonBase agent = (TeamOrPersonBase)getOrMakeAgent(state, authors);
 647                 name.setCombinationAuthorTeam(agent);
 648
 649                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 650                 try {
 651                         if (StringUtils.isNotBlank(determinationLight.rank) ){
 652                                 name.setRank(Rank.getRankByNameOrAbbreviation(determinationLight.rank, nc, true));
 653                         }
 654                 } catch (UnknownCdmTypeException e) {
 655                         String message = "Rank not found: %s: ";
 656                         message = String.format(message, determinationLight.rank);
 657                         logger.warn(message);
 658                 }
 659                 if (StringUtils.isBlank(name.getInfraSpecificEpithet()) && StringUtils.isNotBlank(name.getSpecificEpithet() )){
 660                         name.setRank(Rank.SPECIES());
 661                 }
 662                 if (StringUtils.isBlank(name.getSpecificEpithet()) && StringUtils.isNotBlank(name.getGenusOrUninomial() )){
 663                         name.setRank(Rank.SPECIES());
 664                 }
 665                 if (StringUtils.isBlank(name.getTitleCache())){
 666                         //TODO test
 667                         name.setTitleCache(determinationLight.fullName, true);
 668                 }
 669                 return name;
 670         }
 671
 672         private TaxonNameBase findBestMatchingName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 673
 674                 NonViralName name = makeTaxonName(state, determinationLight);
 675                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 676
 677                 //TODO
 678                 List<TaxonNameBase> matchingNames = getNameService().findByName(null, titleCache, MatchMode.EXACT, null, null, null, null, null).getRecords();
 679                 if (matchingNames.size() > 0){
 680                         return matchingNames.get(0);
 681                 } else if (matchingNames.size() > 0){
 682                         logger.warn("Get best matching taxon name not yet fully implemeted for specimen import");
 683                         return matchingNames.get(0);
 684                 }else{
 685                         return null;
 686                 }
 687
 688         }
 689
 690
 691         private DeterminationEvent makeDeterminationEvent(SpecimenCdmExcelImportState state, DeterminationLight determination, Taxon taxon) {
 692                 DeterminationEvent event = DeterminationEvent.NewInstance();
 693                 //taxon
 694                 event.setTaxon(taxon);
 695
 696                 //date
 697                 TimePeriod date = TimePeriod.parseString(determination.determinedWhen);
 698                 event.setTimeperiod(date);
 699                 //by
 700                 //FIXME bracketAuthors and teams not yet implemented!!!
 701                 List<String> authors = new ArrayList<String>();
 702                 if (StringUtils.isNotBlank(determination.determinedBy)){
 703                         authors.add(determination.determinedBy);
 704                 }
 705                 TeamOrPersonBase<?> actor = getOrMakeAgent(state, authors);
 706                 TeamOrPersonBase<?> secAuthor = taxon.getSec() == null ? null : taxon.getSec().getAuthorTeam();
 707                 if (actor != null && secAuthor != null & secAuthor.getTitleCache().equals(actor.getTitleCache()) && secAuthor.getNomenclaturalTitle().equals(actor.getNomenclaturalTitle())) {
 708                         actor = secAuthor;
 709                 }
 710
 711                 event.setActor(actor);
 712
 713                 //TODO
 714                 if (StringUtils.isNotBlank(determination.modifier)){
 715                         logger.warn("DeterminationModifiers not yet implemented for specimen import");
 716                 }
 717 //              DeterminationModifier modifier = DeterminationModifier.NewInstance(term, label, labelAbbrev);
 718 //              determination.modifier;
 719                 //notes
 720                 if (StringUtils.isNotEmpty(determination.notes)){
 721                         Annotation annotation = Annotation.NewInstance(determination.notes, AnnotationType.EDITORIAL(), Language.DEFAULT());
 722                         event.addAnnotation(annotation);
 723                 }
 724
 725                 return event;
 726         }
 727
 728         private TaxonDescription getTaxonDescription(Taxon taxon) {
 729                 TaxonDescription desc = this.getTaxonDescription(taxon, ! IMAGE_GALLERY, CREATE);
 730                 return desc;
 731         }
 732
 733         private TeamOrPersonBase<?> getOrMakeAgent(SpecimenCdmExcelImportState state, List<String> agents) {
 734                 if (agents.size() == 0){
 735                         return null;
 736                 }else if (agents.size() == 1){
 737                         return getOrMakePerson(state, agents.get(0));
 738                 }else{
 739                         return getOrMakeTeam(state, agents);
 740                 }
 741         }
 742
 743         private Person getOrMakePrimaryCollector(DerivedUnitFacade facade, String primaryCollector, SpecimenCdmExcelImportState state) {
 744                 if (StringUtils.isBlank(primaryCollector)){
 745                         return null;
 746                 }
 747                 AgentBase<?> collector = facade.getCollector();
 748                 List<Person> collectors = new ArrayList<Person>();
 749                 if (collector.isInstanceOf(Team.class) ){
 750                         Team team = CdmBase.deproxy(collector, Team.class);
 751                         collectors.addAll(team.getTeamMembers());
 752                 }else if (collector.isInstanceOf(Person.class)){
 753                         collectors.add(CdmBase.deproxy(collector, Person.class));
 754                 }else{
 755                         throw new IllegalStateException("Unknown subclass of agentbase: " + collector.getClass().getName() );
 756                 }
 757                 for (Person person :collectors){
 758                         if (primaryCollector.equalsIgnoreCase(person.getTitleCache())){
 759                                 return person;
 760                         }
 761                         if (primaryCollector.equalsIgnoreCase(person.getNomenclaturalTitle())){
 762                                 return person;
 763                         }
 764                 }
 765                 String message = "Primary Agent '%s' could not be determined in collector(s) in line %d";
 766                 message = String.format(message, primaryCollector, state.getCurrentLine());
 767                 logger.warn(message);
 768                 return null;
 769         }
 770
 771         private Team getOrMakeTeam(SpecimenCdmExcelImportState state, List<String> agents) {
 772                 String key = CdmUtils.concat("_", agents.toArray(new String[0]));
 773
 774                 Team result = state.getTeam(key);
 775                 if (result == null){
 776                         result = Team.NewInstance();
 777                         for (String member : agents){
 778                                 Person person = getOrMakePerson(state, member);
 779                                 result.addTeamMember(person);
 780                         }
 781                         state.putTeam(key, result);
 782                 }
 783                 return result;
 784         }
 785
 786         private Person getOrMakePerson(SpecimenCdmExcelImportState state, String value) {
 787                 Person result = state.getPerson(value);
 788                 if (result == null){
 789                         result = Person.NewInstance();
 790                         result.setTitleCache(value, true);
 791                         state.putPerson(value, result);
 792                 }
 793                 return result;
 794         }
 795
 796         private Reference<?> getOrMakeReference(SpecimenCdmExcelImportState state, String value) {
 797                 Reference<?> result = state.getReference(value);
 798                 if (result == null){
 799                         result = ReferenceFactory.newGeneric();
 800                         result.setTitleCache(value, true);
 801                         state.putReference(value, result);
 802                 }
 803                 return result;
 804         }
 805
 806
 807
 808         private Collection getOrMakeCollection(SpecimenCdmExcelImportState state, String collectionCode, String collectionString) {
 809                 Collection result = state.getCollection(collectionCode);
 810                 if (result == null){
 811                         result = Collection.NewInstance();
 812                         result.setCode(collectionCode);
 813                         result.setName(collectionString);
 814                         state.putCollection(collectionCode, result);
 815                 }
 816                 return result;
 817         }
 818
 819
 820         private TaxonNameBase<?, ?> getTaxonName(SpecimenCdmExcelImportState state, String name) {
 821                 TaxonNameBase<?,?> result = null;
 822                 result = state.getName(name);
 823                 if (result != null){
 824                         return result;
 825                 }
 826                 List<TaxonNameBase<?,?>> list = getNameService().findNamesByTitle(name);
 827                 //TODO better strategy to find best name, e.g. depending on the classification it is used in
 828                 if (! list.isEmpty()){
 829                         result = list.get(0);
 830                 }
 831                 if (result == null){
 832                         NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
 833                         NomenclaturalCode code = state.getConfig().getNomenclaturalCode();
 834                         result = parser.parseFullName(name, code, null);
 835
 836                 }
 837                 if (result != null){
 838                         state.putName(name, result);
 839                 }
 840                 return result;
 841         }
 842
 843         private SpecimenTypeDesignationStatus getSpecimenTypeStatus(SpecimenCdmExcelImportState state, String key)  {
 844                 SpecimenTypeDesignationStatus result = null;
 845                 try {
 846                         result = state.getTransformer().getSpecimenTypeDesignationStatusByKey(key);
 847                         if (result == null){
 848                                 String message = "Type status not recognized for %s in line %d";
 849                                 message = String.format(message, key, state.getCurrentLine());
 850                                 logger.warn(message);
 851                         }
 852                         return result;
 853                 } catch (UndefinedTransformerMethodException e) {
 854                         throw new RuntimeException("getSpecimenTypeDesignationStatusByKey not yet implemented");
 855                 }
 856
 857
 858         }
 859
 860
 861         private void handleExactLocation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 862
 863                 //reference system
 864                 ReferenceSystem refSys = null;
 865                 if (StringUtils.isNotBlank(row.getReferenceSystem())){
 866                         String strRefSys = row.getReferenceSystem().trim().replaceAll("\\s", "");
 867                         UUID refUuid;
 868                         try {
 869                                 refSys = state.getTransformer().getReferenceSystemByKey(strRefSys);
 870                                 if (refSys == null){
 871                                         refUuid = state.getTransformer().getReferenceSystemUuid(strRefSys);
 872                                         if (refUuid == null){
 873                                                 String message = "Unknown reference system %s in line %d";
 874                                                 message = String.format(message, strRefSys, state.getCurrentLine());
 875                                                 logger.warn(message);
 876                                         }
 877                                         refSys = getReferenceSystem(state, refUuid, strRefSys, strRefSys, strRefSys, null);
 878                                 }
 879
 880                         } catch (UndefinedTransformerMethodException e) {
 881                                 throw new RuntimeException(e);
 882                         }
 883                 }
 884
 885
 886
 887                 // lat/ long /error
 888                 try {
 889                         String longitude = row.getLongitude();
 890                         String latitude = row.getLatitude();
 891                         Integer errorRadius = null;
 892                         if (StringUtils.isNotBlank(row.getErrorRadius())){
 893                                 try {
 894                                         errorRadius = Integer.valueOf(row.getErrorRadius());
 895                                 } catch (NumberFormatException e) {
 896                                         String message = "Error radius %s could not be transformed to Integer in line %d";
 897                                         message = String.format(message, row.getErrorRadius(), state.getCurrentLine());
 898                                         logger.warn(message);
 899                                 }
 900                         }
 901                         //all
 902                         facade.setExactLocationByParsing(longitude, latitude, refSys, errorRadius);
 903                 } catch (ParseException e) {
 904                         String message = "Problems when parsing exact location for line %d";
 905                         message = String.format(message, state.getCurrentLine());
 906                         logger.warn(message);
 907
 908                 }
 909
 910
 911
 912         }
 913
 914
 915         /*
 916          * Set the current Country
 917          * Search in the DB if the isoCode is known
 918          * If not, search if the country name is in the DB
 919          * If not, create a new Label with the Level Country
 920          * @param iso: the country iso code
 921          * @param fullName: the country's full name
 922          * @param app: the CDM application controller
 923          */
 924         private void handleCountry(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 925
 926                 if (StringUtils.isNotBlank(row.getIsoCountry())){
 927                         NamedArea country = getOccurrenceService().getCountryByIso(row.getIsoCountry());
 928                         if (country != null){
 929                                 facade.setCountry(country);
 930                                 return;
 931                         }
 932                 }
 933                 if (StringUtils.isNotBlank(row.getCountry())){
 934                         List<WaterbodyOrCountry> countries = getOccurrenceService().getWaterbodyOrCountryByName(row.getCountry());
 935                         if (countries.size() >0){
 936                                 facade.setCountry(countries.get(0));
 937                         }else{
 938                                 UUID uuid = UUID.randomUUID();
 939                                 String label = row.getCountry();
 940                                 String text = row.getCountry();
 941                                 String labelAbbrev = null;
 942                                 NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
 943                                 NamedAreaLevel level = NamedAreaLevel.COUNTRY();
 944                                 NamedArea newCountry = this.getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level);
 945                                 facade.setCountry(newCountry);
 946                         }
 947                 }
 948         }
 949
 950         @Override
 951         protected void secondPass(SpecimenCdmExcelImportState state) {
 952                 //no second path defined yet
 953                 return;
 954         }
 955
 956
 957         @Override
 958         protected String getWorksheetName() {
 959                 return WORKSHEET_NAME;
 960         }
 961
 962         @Override
 963         protected boolean needsNomenclaturalCode() {
 964                 return false;
 965         }
 966
 967
 968         /* (non-Javadoc)
 969          * @see eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase#createDataHolderRow()
 970          */
 971         @Override
 972         protected SpecimenRow createDataHolderRow() {
 973                 return new SpecimenRow();
 974         }
 975
 976
 977
 978
 979         /* (non-Javadoc)
 980          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 981          */
 982         @Override
 983         protected boolean doCheck(SpecimenCdmExcelImportState state) {
 984                 logger.warn("Validation not yet implemented for " + this.getClass().getSimpleName());
 985                 return true;
 986         }
 987
 988
 989
 990         @Override
 991         protected boolean isIgnore(SpecimenCdmExcelImportState state) {
 992                 return !state.getConfig().isDoSpecimen();
 993         }
 994
 995
 996 }