cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/excel/in/SpecimenCdmExcelImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen.excel.in;
  11
  12 import java.text.ParseException;
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.List;
  16 import java.util.Set;
  17 import java.util.UUID;
  18
  19 import org.apache.commons.lang.StringUtils;
  20 import org.apache.log4j.Logger;
  21 import org.springframework.stereotype.Component;
  22
  23 import sun.security.util.DerEncoder;
  24
  25 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  26 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  27 import eu.etaxonomy.cdm.api.service.config.MatchingTaxonConfigurator;
  28 import eu.etaxonomy.cdm.common.CdmUtils;
  29 import eu.etaxonomy.cdm.io.common.CdmImportBase.TermMatchMode;
  30 import eu.etaxonomy.cdm.io.common.ICdmIO;
  31 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  32 import eu.etaxonomy.cdm.io.excel.common.ExcelImporterBase;
  33 import eu.etaxonomy.cdm.io.specimen.excel.in.SpecimenRow.DeterminationLight;
  34 import eu.etaxonomy.cdm.io.specimen.excel.in.SpecimenRow.PostfixTerm;
  35 import eu.etaxonomy.cdm.model.agent.AgentBase;
  36 import eu.etaxonomy.cdm.model.agent.Person;
  37 import eu.etaxonomy.cdm.model.agent.Team;
  38 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  39 import eu.etaxonomy.cdm.model.common.Annotation;
  40 import eu.etaxonomy.cdm.model.common.AnnotationType;
  41 import eu.etaxonomy.cdm.model.common.Extension;
  42 import eu.etaxonomy.cdm.model.common.ExtensionType;
  43 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  44 import eu.etaxonomy.cdm.model.common.Language;
  45 import eu.etaxonomy.cdm.model.common.TimePeriod;
  46 import eu.etaxonomy.cdm.model.description.Feature;
  47 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  48 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  49 import eu.etaxonomy.cdm.model.location.NamedArea;
  50 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
  51 import eu.etaxonomy.cdm.model.location.NamedAreaType;
  52 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  53 import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
  54 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  55 import eu.etaxonomy.cdm.model.name.NonViralName;
  56 import eu.etaxonomy.cdm.model.name.Rank;
  57 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  58 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  59 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  60 import eu.etaxonomy.cdm.model.occurrence.Collection;
  61 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
  62 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  63 import eu.etaxonomy.cdm.model.reference.Reference;
  64 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  65 import eu.etaxonomy.cdm.model.taxon.Taxon;
  66 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  67 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  68 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  69
  70 /**
  71  * @author a.mueller
  72  * @created 10.05.2011
  73  * @version 1.0
  74  */
  75 @Component
  76 public class SpecimenCdmExcelImport  extends ExcelImporterBase<SpecimenCdmExcelImportState>  implements ICdmIO<SpecimenCdmExcelImportState> {
  77         private static final Logger logger = Logger.getLogger(SpecimenCdmExcelImport.class);
  78
  79         private static final String WORKSHEET_NAME = "Specimen";
  80
  81         private static final String UUID_COLUMN = "(?i)(UUID)";
  82         private static final String BASIS_OF_RECORD_COLUMN = "(?i)(BasisOfRecord)";
  83         private static final String COUNTRY_COLUMN = "(?i)(Country)";
  84         private static final String AREA_COLUMN = "(?i)(Area)";
  85         private static final String ISO_COUNTRY_COLUMN = "(?i)(ISOCountry|CountryCode)";
  86         private static final String LOCALITY_COLUMN = "(?i)(Locality)";
  87         private static final String ALTITUDE_COLUMN = "(?i)(AbsoluteElevation|Altitude)";
  88         private static final String ALTITUDE_MAX_COLUMN = "(?i)(AbsoluteElevation|Altitude)Max(imum)?";
  89         private static final String COLLECTION_DATE_COLUMN = "(?i)(CollectionDate)";
  90         private static final String COLLECTION_DATE_END_COLUMN = "(?i)(CollectionDateEnd)";
  91         private static final String COLLECTOR_COLUMN = "(?i)(Collector)";
  92         private static final String LONGITUDE_COLUMN = "(?i)(Longitude)";
  93         private static final String LATITUDE_COLUMN = "(?i)(Latitude)";
  94         private static final String REFERENCE_SYSTEM_COLUMN = "(?i)(ReferenceSystem)";
  95         private static final String ERROR_RADIUS_COLUMN = "(?i)(ErrorRadius)";
  96
  97
  98         private static final String COLLECTORS_NUMBER_COLUMN = "(?i)((Collectors|Field)Number)";
  99         private static final String ECOLOGY_COLUMN = "(?i)(Ecology|Habitat)";
 100         private static final String PLANT_DESCRIPTION_COLUMN = "(?i)(PlantDescription)";
 101         private static final String FIELD_NOTES_COLUMN = "(?i)(FieldNotes)";
 102         private static final String SEX_COLUMN = "(?i)(Sex)";
 103
 104
 105         private static final String ACCESSION_NUMBER_COLUMN = "(?i)(AccessionNumber)";
 106         private static final String BARCODE_COLUMN = "(?i)(Barcode)";
 107         private static final String COLLECTION_CODE_COLUMN = "(?i)(CollectionCode)";
 108         private static final String COLLECTION_COLUMN = "(?i)(Collection)";
 109         private static final String UNIT_NOTES_COLUMN = "(?i)((Unit)?Notes)";
 110
 111
 112         private static final String TYPE_CATEGORY_COLUMN = "(?i)(TypeCategory)";
 113         private static final String TYPIFIED_NAME_COLUMN = "(?i)(TypifiedName|TypeOf)";
 114
 115
 116         private static final String SOURCE_COLUMN = "(?i)(Source)";
 117         private static final String ID_IN_SOURCE_COLUMN = "(?i)(IdInSource)";
 118
 119
 120         private static final String RANK_COLUMN = "(?i)(Rank)";
 121         private static final String FULL_NAME_COLUMN = "(?i)(FullName)";
 122         private static final String FAMILY_COLUMN = "(?i)(Family)";
 123         private static final String GENUS_COLUMN = "(?i)(Genus)";
 124         private static final String SPECIFIC_EPITHET_COLUMN = "(?i)(SpecificEpi(thet)?)";
 125         private static final String INFRASPECIFIC_EPITHET_COLUMN = "(?i)(InfraSpecificEpi(thet)?)";
 126         private static final String DETERMINATION_AUTHOR_COLUMN = "(?i)(Author)";
 127         private static final String DETERMINATION_MODIFIER_COLUMN = "(?i)(DeterminationModifier)";
 128         private static final String DETERMINED_BY_COLUMN = "(?i)(DeterminationBy)";
 129         private static final String DETERMINED_WHEN_COLUMN = "(?i)(DeterminationWhen)";
 130         private static final String DETERMINATION_NOTES_COLUMN = "(?i)(DeterminationNote)";
 131         private static final String EXTENSION_COLUMN = "(?i)(Ext(ension)?)";
 132
 133         private static final String IGNORE_COLUMN = "(?i)(Ignore|Not)";
 134
 135
 136         public SpecimenCdmExcelImport() {
 137                 super();
 138         }
 139
 140
 141         @Override
 142         protected boolean analyzeRecord(HashMap<String, String> record, SpecimenCdmExcelImportState state) {
 143                 boolean success = true;
 144         Set<String> keys = record.keySet();
 145
 146         SpecimenRow row = new SpecimenRow();
 147         state.setSpecimenRow(row);
 148
 149         for (String originalKey: keys) {
 150                 Integer index = 0;
 151                 String postfix = null;
 152                 String indexedKey = CdmUtils.removeDuplicateWhitespace(originalKey.trim()).toString();
 153                 String[] split = indexedKey.split("_");
 154                 String key = split[0];
 155                 if (split.length > 1){
 156                         for (int i = 1 ; i < split.length ; i++ ){
 157                                 String indexString = split[i];
 158                                 if (isInteger(indexString)){
 159                                         index = Integer.valueOf(indexString);
 160                                 }else{
 161                                         postfix = split[i];
 162                                 }
 163                         }
 164                 }
 165
 166                 String value = (String) record.get(indexedKey);
 167                 if (! StringUtils.isBlank(value)) {
 168                         if (logger.isDebugEnabled()) { logger.debug(key + ": " + value); }
 169                         value = CdmUtils.removeDuplicateWhitespace(value.trim()).toString();
 170                 }else{
 171                         continue;
 172                 }
 173
 174                 if (key.matches(UUID_COLUMN)) {
 175                         row.setUuid(UUID.fromString(value)); //VALIDATE UUID
 176                         } else if(key.matches(BASIS_OF_RECORD_COLUMN)) {
 177                                 row.setBasisOfRecord(value);
 178                         } else if(key.matches(COUNTRY_COLUMN)) {
 179                                 row.setCountry(value);
 180                         } else if(key.matches(ISO_COUNTRY_COLUMN)) {
 181                                 row.setIsoCountry(value);
 182                         } else if(key.matches(LOCALITY_COLUMN)) {
 183                                 row.setLocality(value);
 184                         } else if(key.matches(FIELD_NOTES_COLUMN)) {
 185                                 row.setLocality(value);
 186                         } else if(key.matches(ALTITUDE_COLUMN)) {
 187                                 row.setAltitude(value);
 188                         } else if(key.matches(ALTITUDE_MAX_COLUMN)) {
 189                                 row.setAltitudeMax(value);
 190                         } else if(key.matches(COLLECTOR_COLUMN)) {
 191                                 row.putCollector(index, value);
 192                         } else if(key.matches(ECOLOGY_COLUMN)) {
 193                                 row.setEcology(value);
 194                         } else if(key.matches(PLANT_DESCRIPTION_COLUMN)) {
 195                                 row.setPlantDescription(value);
 196                         } else if(key.matches(SEX_COLUMN)) {
 197                                 row.setSex(value);
 198                         } else if(key.matches(COLLECTION_DATE_COLUMN)) {
 199                                 row.setCollectingDate(value);
 200                         } else if(key.matches(COLLECTION_DATE_END_COLUMN)) {
 201                                 row.setCollectingDateEnd(value);
 202                         } else if(key.matches(COLLECTOR_COLUMN)) {
 203                                 row.putCollector(index, value);
 204                         } else if(key.matches(COLLECTORS_NUMBER_COLUMN)) {
 205                                 row.setCollectorsNumber(value);
 206                         } else if(key.matches(LONGITUDE_COLUMN)) {
 207                                 row.setLongitude(value);
 208                         } else if(key.matches(LATITUDE_COLUMN)) {
 209                                 row.setLatitude(value);
 210                         } else if(key.matches(REFERENCE_SYSTEM_COLUMN)) {
 211                                 row.setReferenceSystem(value);
 212                         } else if(key.matches(ERROR_RADIUS_COLUMN)) {
 213                                 row.setErrorRadius(value);
 214                         } else if(key.matches(AREA_COLUMN)) {
 215                                 if (postfix != null){
 216                                         row.addLeveledArea(postfix, value);
 217                                 }else{
 218                                         logger.warn("Not yet implemented");
 219                                 }
 220
 221
 222
 223                         } else if(key.matches(ACCESSION_NUMBER_COLUMN)) {
 224                                 row.setLocality(value);
 225                         } else if(key.matches(BARCODE_COLUMN)) {
 226                                 row.setBarcode(value);
 227                         } else if(key.matches(UNIT_NOTES_COLUMN)) {
 228                                 row.putUnitNote(index, value);
 229
 230
 231                         } else if(key.matches(FAMILY_COLUMN)) {
 232                                 row.putDeterminationFamily(index, value);
 233                         } else if(key.matches(GENUS_COLUMN)) {
 234                                 row.putDeterminationGenus(index, value);
 235                         } else if(key.matches(SPECIFIC_EPITHET_COLUMN)) {
 236                                 row.putDeterminationSpeciesEpi(index, value);
 237                         } else if(key.matches(INFRASPECIFIC_EPITHET_COLUMN)) {
 238                                 row.putDeterminationInfraSpeciesEpi(index, value);
 239                         } else if(key.matches(RANK_COLUMN)) {
 240                                 row.putDeterminationRank(index, value);
 241                         } else if(key.matches(FULL_NAME_COLUMN)) {
 242                                 row.putDeterminationFullName(index, value);
 243                         } else if(key.matches(DETERMINATION_AUTHOR_COLUMN)) {
 244                                 row.putDeterminationAuthor(index, value);
 245                         } else if(key.matches(DETERMINATION_MODIFIER_COLUMN)) {
 246                                 row.putDeterminationDeterminationModifier(index, value);
 247                         } else if(key.matches(DETERMINATION_NOTES_COLUMN)) {
 248                                 row.putDeterminationDeterminationNotes(index, value);
 249                         } else if(key.matches(DETERMINED_BY_COLUMN)) {
 250                                 row.putDeterminationDeterminedBy(index, value);
 251                         } else if(key.matches(DETERMINED_WHEN_COLUMN)) {
 252                                 row.putDeterminationDeterminedWhen(index, value);
 253
 254                         } else if(key.matches(COLLECTION_CODE_COLUMN)) {
 255                                 row.setCollectionCode(value);
 256                         } else if(key.matches(COLLECTION_COLUMN)) {
 257                                 row.setCollection(value);
 258
 259                         } else if(key.matches(TYPE_CATEGORY_COLUMN)) {
 260                                 row.putTypeCategory(index, getSpecimenTypeStatus(state, value));
 261                         } else if(key.matches(TYPIFIED_NAME_COLUMN)) {
 262                                 row.putTypifiedName(index, getTaxonName(state, value));
 263
 264
 265                         } else if(key.matches(SOURCE_COLUMN)) {
 266                                 row.putSourceReference(index, getOrMakeReference(state, value));
 267                         } else if(key.matches(ID_IN_SOURCE_COLUMN)) {
 268                                 row.putIdInSource(index, value);
 269                         } else if(key.matches(EXTENSION_COLUMN)) {
 270                                 if (postfix != null){
 271                                         row.addExtension(postfix, value);
 272                                 }else{
 273                                         logger.warn("Extension without postfix not yet implemented");
 274                                 }
 275
 276                         } else if(key.matches(IGNORE_COLUMN)) {
 277                                 logger.debug("Ignored column" + originalKey);
 278                         }else {
 279                                 success = false;
 280                                 logger.error("Unexpected column header " + originalKey);
 281                         }
 282         }
 283         return success;
 284         }
 285
 286         @Override
 287         protected boolean firstPass(SpecimenCdmExcelImportState state) {
 288                 SpecimenRow row = state.getSpecimenRow();
 289
 290                 //basis of record
 291                 DerivedUnitType type = DerivedUnitType.valueOf2(row.getBasisOfRecord());
 292                 if (type == null){
 293                         String message = "%s is not a valid BasisOfRecord. 'Unknown' is used instead.";
 294                         message = String.format(message, row.getBasisOfRecord());
 295                         logger.warn(message);
 296                         type = DerivedUnitType.DerivedUnit;
 297                 }
 298                 DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(type);
 299
 300                 //country
 301                 handleCountry(facade, row, state);
 302                 handleAreas(facade,row, state);
 303
 304                 facade.setGatheringPeriod(getTimePeriod(row.getCollectingDate(), row.getCollectingDateEnd()));
 305                 facade.setLocality(row.getLocality());
 306                 facade.setFieldNotes(row.getFieldNotes());
 307                 facade.setFieldNumber(row.getCollectorsNumber());
 308                 facade.setEcology(row.getEcology());
 309                 facade.setPlantDescription(row.getPlantDescription());
 310 //              facade.setSex(row.get)
 311                 handleExactLocation(facade, row, state);
 312                 facade.setCollector(getOrMakeAgent(state, row.getCollectors()));
 313                 handleAbsoluteElevation(facade, row, state);
 314
 315
 316                 //derivedUnit
 317                 facade.setBarcode(row.getBarcode());
 318                 facade.setAccessionNumber(row.getAccessionNumber());
 319                 facade.setCollection(getOrMakeCollection(state, row.getCollectionCode(), row.getCollection()));
 320                 for (IdentifiableSource source : row.getSources()){
 321                         facade.addSource(source);
 322                 }
 323                 for (SpecimenTypeDesignation designation : row.getTypeDesignations()){
 324                         //FIXME
 325 //                      facade.innerDerivedUnit().addSpecimenTypeDesignation(designation);
 326                 }
 327                 handleDeterminations(state, row, facade);
 328                 handleExtensions(facade,row, state);
 329                 for (String note : row.getUnitNotes()){
 330                         Annotation annotation = Annotation.NewInstance(note, AnnotationType.EDITORIAL(), Language.DEFAULT());
 331                         facade.addAnnotation(annotation);
 332                 }
 333
 334                 //save
 335                 getOccurrenceService().save(facade.innerDerivedUnit());
 336                 return true;
 337         }
 338
 339         private void handleAbsoluteElevation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 340                 //altitude
 341
 342                 try {
 343                         String altitude = row.getAltitude();
 344                         if (StringUtils.isBlank(altitude)){
 345                                 return;
 346                         }
 347 //                      if (altitude.endsWith(".0")){
 348 //                              altitude = altitude.substring(0, altitude.length() -2);
 349 //                      }
 350                         int value = Integer.valueOf(altitude);
 351                         facade.setAbsoluteElevation(value);
 352                 } catch (NumberFormatException e) {
 353                         String message = "Absolute elevation / Altitude '%s' is not an integer number in line %d";
 354                         message = String.format(message, row.getAltitude(), state.getCurrentLine());
 355                         logger.warn(message);
 356                         return;
 357                 }
 358
 359                 //max
 360
 361                 try {
 362                         String max = row.getAltitudeMax();
 363                         if (StringUtils.isBlank(max)){
 364                                 return;
 365                         }
 366 //                      if (max.endsWith(".0")){
 367 //                              max = max.substring(0, max.length() -2);
 368 //                      }
 369                         int value = Integer.valueOf(max);
 370                         //TODO avoid unequal distance
 371                         int min = facade.getAbsoluteElevation();
 372                         if ( (value - min) % 2 == 1 ){
 373                                 String message = "Altitude min-max difference ist not equal. Max reduced by 1 in line %d";
 374                                 message = String.format(message, state.getCurrentLine());
 375                                 logger.warn(message);
 376                                 value--;
 377                         }
 378                         facade.setAbsoluteElevationRange(min, value);
 379                 } catch (NumberFormatException e) {
 380                         String message = "Absolute elevation / Altitude maximum '%s' is not an integer number in line %d";
 381                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 382                         logger.warn(message);
 383                         return;
 384                 }catch (Exception e){
 385                         String message = "Error occurred when trying to write Absolute elevation / Altitude maximum '%s' in line %d";
 386                         message = String.format(message, row.getAltitudeMax(), state.getCurrentLine());
 387                         logger.warn(message);
 388                         return;
 389
 390                 }
 391
 392
 393         }
 394
 395
 396         private void handleExtensions(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 397                 List<PostfixTerm> extensions = row.getExtensions();
 398
 399                 for (PostfixTerm exType : extensions){
 400                         ExtensionType extensionType = state.getPostfixExtensionType(exType.postfix);
 401
 402                         Extension extension = Extension.NewInstance();
 403                         extension.setType(extensionType);
 404                         extension.setValue(exType.term);
 405                         facade.innerDerivedUnit().addExtension(extension);
 406                 }
 407
 408         }
 409
 410
 411         private void handleAreas(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 412                 List<PostfixTerm> areas = row.getLeveledAreas();
 413
 414                 for (PostfixTerm lArea : areas){
 415                         String description = lArea.term;
 416                         String abbrev = lArea.term;
 417                         NamedAreaType type = null;
 418                         String key = lArea.postfix + "_" + lArea.term;
 419                         UUID areaUuid = state.getArea(key);
 420                         NamedAreaLevel level = state.getPostfixLevel(lArea.postfix);
 421
 422                         TermMatchMode matchMode = state.getConfig().getAreaMatchMode();
 423                         NamedArea area = getNamedArea(state, areaUuid, lArea.term, description, abbrev, type, level, null, matchMode);
 424                         facade.addCollectingArea(area);
 425                         if (areaUuid == null){
 426                                 state.putArea(key, area.getUuid());
 427                         }
 428                 }
 429         }
 430
 431
 432         /**
 433          * @param state
 434          * @param row
 435          * @param facade
 436          */
 437         private void handleDeterminations(SpecimenCdmExcelImportState state,SpecimenRow row, DerivedUnitFacade facade) {
 438                 boolean isFirstDetermination = true;
 439                 for (DeterminationLight determinationLight : row.getDetermination()){
 440                         Taxon taxon = findBestMatchingTaxon(state, determinationLight, state.getConfig().isCreateTaxonIfNotExists());
 441                         TaxonNameBase<?,?> name = findBestMatchingName(state, determinationLight);
 442                         if (taxon != null){
 443                                 getTaxonService().saveOrUpdate(taxon);
 444                                 if (state.getConfig().isMakeIndividualAssociations() && taxon != null){
 445                                         IndividualsAssociation indivAssociciation = IndividualsAssociation.NewInstance();
 446                                         DerivedUnitBase<?> du = facade.innerDerivedUnit();
 447                                         indivAssociciation.setAssociatedSpecimenOrObservation(du);
 448                                         getTaxonDescription(taxon).addElement(indivAssociciation);
 449                                         Feature feature = Feature.INDIVIDUALS_ASSOCIATION();
 450                                         if (facade.getType().equals(DerivedUnitType.Specimen)){
 451                                                 feature = Feature.SPECIMEN();
 452                                         }else if (facade.getType().equals(DerivedUnitType.Observation)){
 453                                                 feature = Feature.OBSERVATION();
 454                                         }
 455                                         if (state.getConfig().isUseMaterialsExaminedForIndividualsAssociations()){
 456                                                 feature = Feature.MATERIALS_EXAMINED();
 457                                         }
 458
 459                                         indivAssociciation.setFeature(feature);
 460                                 }
 461                                 if (state.getConfig().isDeterminationsAreDeterminationEvent()){
 462                                         DeterminationEvent detEvent = makeDeterminationEvent(state, determinationLight, taxon);
 463                                         detEvent.setPreferredFlag(isFirstDetermination);
 464                                         facade.addDetermination(detEvent);
 465                                 }
 466                         }
 467                         if (name != null){
 468                                 if (isFirstDetermination && state.getConfig().isFirstDeterminationIsStoredUnder()){
 469                                         facade.setStoredUnder(name);
 470                                 }
 471                         }
 472                         isFirstDetermination = false;
 473                 }
 474         }
 475
 476         /**
 477          * This method tries to find the best matching taxon depending on the import configuration,
 478          * the taxon name information and the concept information available.
 479          *
 480          *
 481          * @param state
 482          * @param determinationLight
 483          * @param createIfNotExists
 484          * @return
 485          */
 486         private Taxon findBestMatchingTaxon(SpecimenCdmExcelImportState state, DeterminationLight determinationLight, boolean createIfNotExists) {
 487                 NonViralName<?> name = makeTaxonName(state, determinationLight);
 488
 489                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 490
 491                 if (! StringUtils.isBlank(titleCache)){
 492                         MatchingTaxonConfigurator matchConfigurator = MatchingTaxonConfigurator.NewInstance();
 493                         matchConfigurator.setTaxonNameTitle(titleCache);
 494                         matchConfigurator.setIncludeSynonyms(false);
 495                         Taxon taxon = getTaxonService().findBestMatchingTaxon(matchConfigurator);
 496
 497                         if(taxon == null && createIfNotExists){
 498                                 logger.info("creating new Taxon from TaxonName '" + titleCache+"'");
 499                                 UUID secUuid = null; //TODO
 500                                 Reference<?> sec = null;
 501                                 if (secUuid != null){
 502                                         sec = getReferenceService().find(secUuid);
 503                                 }
 504                                 taxon = Taxon.NewInstance(name, sec);
 505                         }else if (taxon == null){
 506                                 String message = "Taxon '%s' not found in line %d";
 507                                 message = String.format(message, titleCache, state.getCurrentLine());
 508                                 logger.warn(message);
 509                         }
 510                         return taxon;
 511                 }else {
 512                         return null;
 513                 }
 514         }
 515
 516         /**
 517          * @param state
 518          * @param determinationLight
 519          * @param name
 520          * @return
 521          */
 522         private String makeSearchNameTitleCache(SpecimenCdmExcelImportState state, DeterminationLight determinationLight,
 523                                 NonViralName name) {
 524                 String titleCache = determinationLight.fullName;
 525                 if (! state.getConfig().isPreferNameCache() || StringUtils.isBlank(titleCache) ){
 526                         String computedTitleCache = name.getTitleCache();
 527                         if (StringUtils.isNotBlank(computedTitleCache)){
 528                                 titleCache = computedTitleCache;
 529                         }
 530
 531                 }
 532                 return titleCache;
 533         }
 534
 535         /**
 536          * @param state
 537          * @param determinationLight
 538          * @return
 539          */
 540         private NonViralName makeTaxonName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 541                 //TODO correct type by config.nc
 542                 NonViralName name =NonViralName.NewInstance(null);
 543                 name.setGenusOrUninomial(determinationLight.genus);
 544                 name.setSpecificEpithet(determinationLight.speciesEpi);
 545                 name.setInfraSpecificEpithet(determinationLight.infraSpeciesEpi);
 546
 547                 //FIXME bracketAuthors and teams not yet implemented!!!
 548                 List<String> authors = new ArrayList<String>();
 549                 if (StringUtils.isNotBlank(determinationLight.author)){
 550                         authors.add(determinationLight.author);
 551                 }
 552                 TeamOrPersonBase agent = (TeamOrPersonBase)getOrMakeAgent(state, authors);
 553                 name.setCombinationAuthorTeam(agent);
 554
 555                 NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 556                 try {
 557                         if (StringUtils.isNotBlank(determinationLight.rank) ){
 558                                 name.setRank(Rank.getRankByNameOrAbbreviation(determinationLight.rank, nc, true));
 559                         }
 560                 } catch (UnknownCdmTypeException e) {
 561                         String message = "Rank not found: %s: ";
 562                         message = String.format(message, determinationLight.rank);
 563                         logger.warn(message);
 564                 }
 565                 if (StringUtils.isBlank(name.getInfraSpecificEpithet()) && StringUtils.isNotBlank(name.getSpecificEpithet() )){
 566                         name.setRank(Rank.SPECIES());
 567                 }
 568                 if (StringUtils.isBlank(name.getSpecificEpithet()) && StringUtils.isNotBlank(name.getGenusOrUninomial() )){
 569                         name.setRank(Rank.SPECIES());
 570                 }
 571                 if (StringUtils.isBlank(name.getTitleCache())){
 572                         //TODO test
 573                         name.setTitleCache(determinationLight.fullName, true);
 574                 }
 575                 return name;
 576         }
 577
 578         private TaxonNameBase findBestMatchingName(SpecimenCdmExcelImportState state, DeterminationLight determinationLight) {
 579
 580                 NonViralName name = makeTaxonName(state, determinationLight);
 581                 String titleCache = makeSearchNameTitleCache(state, determinationLight, name);
 582
 583                 //TODO
 584                 List<TaxonNameBase> matchingNames = getNameService().findByName(null, titleCache, MatchMode.EXACT, null, null, null, null, null).getRecords();
 585                 if (matchingNames.size() > 0){
 586                         return matchingNames.get(0);
 587                 } else if (matchingNames.size() > 0){
 588                         logger.warn("Get best matching taxon name not yet fully implemeted for specimen import");
 589                         return matchingNames.get(0);
 590                 }else{
 591                         return null;
 592                 }
 593
 594         }
 595
 596
 597         private DeterminationEvent makeDeterminationEvent(SpecimenCdmExcelImportState state, DeterminationLight determination, Taxon taxon) {
 598                 DeterminationEvent event = DeterminationEvent.NewInstance();
 599                 //taxon
 600                 event.setTaxon(taxon);
 601
 602                 //date
 603                 TimePeriod date = TimePeriod.parseString(determination.determinedWhen);
 604                 event.setTimeperiod(date);
 605                 //by
 606                 //FIXME bracketAuthors and teams not yet implemented!!!
 607                 List<String> authors = new ArrayList<String>();
 608                 if (StringUtils.isNotBlank(determination.determinedBy)){
 609                         authors.add(determination.determinedBy);
 610                 }
 611                 AgentBase actor = getOrMakeAgent(state, authors);
 612                 event.setActor(actor);
 613
 614                 //TODO
 615                 if (StringUtils.isNotBlank(determination.modifier)){
 616                         logger.warn("DeterminationModifiers not yet implemented for specimen import");
 617                 }
 618 //              DeterminationModifier modifier = DeterminationModifier.NewInstance(term, label, labelAbbrev);
 619 //              determination.modifier;
 620                 //notes
 621                 Annotation annotation = Annotation.NewInstance(determination.notes, AnnotationType.EDITORIAL(), Language.DEFAULT());
 622                 event.addAnnotation(annotation);
 623
 624                 return event;
 625         }
 626
 627         private TaxonDescription getTaxonDescription(Taxon taxon) {
 628                 TaxonDescription desc = this.getTaxonDescription(taxon, ! IMAGE_GALLERY, CREATE);
 629                 return desc;
 630         }
 631
 632         private AgentBase<?> getOrMakeAgent(SpecimenCdmExcelImportState state, List<String> agents) {
 633                 if (agents.size() == 0){
 634                         return null;
 635                 }else if (agents.size() == 1){
 636                         return getOrMakePerson(state, agents.get(0));
 637                 }else{
 638                         return getOrMakeTeam(state, agents);
 639                 }
 640         }
 641
 642         private Team getOrMakeTeam(SpecimenCdmExcelImportState state, List<String> agents) {
 643                 String key = CdmUtils.concat("_", agents.toArray(new String[0]));
 644
 645                 Team result = state.getTeam(key);
 646                 if (result == null){
 647                         result = Team.NewInstance();
 648                         for (String member : agents){
 649                                 Person person = getOrMakePerson(state, member);
 650                                 result.addTeamMember(person);
 651                         }
 652                         state.putTeam(key, result);
 653                 }
 654                 return result;
 655         }
 656
 657         private Person getOrMakePerson(SpecimenCdmExcelImportState state, String value) {
 658                 Person result = state.getPerson(value);
 659                 if (result == null){
 660                         result = Person.NewInstance();
 661                         result.setTitleCache(value, true);
 662                         state.putPerson(value, result);
 663                 }
 664                 return result;
 665         }
 666
 667         private Reference<?> getOrMakeReference(SpecimenCdmExcelImportState state, String value) {
 668                 Reference<?> result = state.getReference(value);
 669                 if (result == null){
 670                         result = ReferenceFactory.newGeneric();
 671                         result.setTitleCache(value, true);
 672                         state.putReference(value, result);
 673                 }
 674                 return result;
 675         }
 676
 677
 678
 679         private Collection getOrMakeCollection(SpecimenCdmExcelImportState state, String collectionCode, String collectionString) {
 680                 Collection result = state.getCollection(collectionCode);
 681                 if (result == null){
 682                         result = Collection.NewInstance();
 683                         result.setCode(collectionCode);
 684                         result.setName(collectionString);
 685                         state.putCollection(collectionCode, result);
 686                 }
 687                 return result;
 688         }
 689
 690
 691         private TaxonNameBase<?, ?> getTaxonName(SpecimenCdmExcelImportState state, String name) {
 692                 TaxonNameBase<?,?> result = null;
 693                 result = state.getName(name);
 694                 if (result != null){
 695                         return result;
 696                 }
 697                 List<TaxonNameBase<?,?>> list = getNameService().findNamesByTitle(name);
 698                 //TODO better strategy to find best name, e.g. depending on the classification it is used in
 699                 if (! list.isEmpty()){
 700                         result = list.get(0);
 701                 }
 702                 if (result == null){
 703                         NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
 704                         NomenclaturalCode code = state.getConfig().getNomenclaturalCode();
 705                         result = parser.parseFullName(name, code, null);
 706
 707                 }
 708                 if (result != null){
 709                         state.putName(name, result);
 710                 }
 711                 return result;
 712         }
 713
 714         private SpecimenTypeDesignationStatus getSpecimenTypeStatus(SpecimenCdmExcelImportState state, String key)  {
 715                 SpecimenTypeDesignationStatus result = null;
 716                 try {
 717                         result = state.getTransformer().getSpecimenTypeDesignationStatusByKey(key);
 718                         if (result == null){
 719                                 String message = "Type status not recognized for %s in line %d";
 720                                 message = String.format(message, key, state.getCurrentLine());
 721                                 logger.warn(message);
 722                         }
 723                         return result;
 724                 } catch (UndefinedTransformerMethodException e) {
 725                         throw new RuntimeException("getSpecimenTypeDesignationStatusByKey not yet implemented");
 726                 }
 727
 728
 729         }
 730
 731
 732         private void handleExactLocation(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 733
 734                 //reference system
 735                 ReferenceSystem refSys = null;
 736                 if (StringUtils.isNotBlank(row.getReferenceSystem())){
 737                         String strRefSys = row.getReferenceSystem().trim().replaceAll("\\s", "");
 738                         UUID refUuid;
 739                         try {
 740                                 refSys = state.getTransformer().getReferenceSystemByKey(strRefSys);
 741                                 if (refSys == null){
 742                                         refUuid = state.getTransformer().getReferenceSystemUuid(strRefSys);
 743                                         if (refUuid == null){
 744                                                 String message = "Unknown reference system %s in line %d";
 745                                                 message = String.format(message, strRefSys, state.getCurrentLine());
 746                                                 logger.warn(message);
 747                                         }
 748                                         refSys = getReferenceSystem(state, refUuid, strRefSys, strRefSys, strRefSys, null);
 749                                 }
 750
 751                         } catch (UndefinedTransformerMethodException e) {
 752                                 throw new RuntimeException(e);
 753                         }
 754                 }
 755
 756
 757
 758                 // lat/ long /error
 759                 try {
 760                         String longitude = row.getLongitude();
 761                         String latitude = row.getLatitude();
 762                         Integer errorRadius = null;
 763                         if (StringUtils.isNotBlank(row.getErrorRadius())){
 764                                 try {
 765                                         errorRadius = Integer.valueOf(row.getErrorRadius());
 766                                 } catch (NumberFormatException e) {
 767                                         String message = "Error radius %s could not be transformed to Integer in line %d";
 768                                         message = String.format(message, row.getErrorRadius(), state.getCurrentLine());
 769                                         logger.warn(message);
 770                                 }
 771                         }
 772                         //all
 773                         facade.setExactLocationByParsing(longitude, latitude, refSys, errorRadius);
 774                 } catch (ParseException e) {
 775                         String message = "Problems when parsing exact location for line %d";
 776                         message = String.format(message, state.getCurrentLine());
 777                         logger.warn(message);
 778
 779                 }
 780
 781
 782
 783         }
 784
 785
 786         /*
 787          * Set the current Country
 788          * Search in the DB if the isoCode is known
 789          * If not, search if the country name is in the DB
 790          * If not, create a new Label with the Level Country
 791          * @param iso: the country iso code
 792          * @param fullName: the country's full name
 793          * @param app: the CDM application controller
 794          */
 795         private void handleCountry(DerivedUnitFacade facade, SpecimenRow row, SpecimenCdmExcelImportState state) {
 796
 797                 if (StringUtils.isNotBlank(row.getIsoCountry())){
 798                         NamedArea country = getOccurrenceService().getCountryByIso(row.getIsoCountry());
 799                         if (country != null){
 800                                 facade.setCountry(country);
 801                                 return;
 802                         }
 803                 }
 804                 if (StringUtils.isNotBlank(row.getCountry())){
 805                         List<WaterbodyOrCountry> countries = getOccurrenceService().getWaterbodyOrCountryByName(row.getCountry());
 806                         if (countries.size() >0){
 807                                 facade.setCountry(countries.get(0));
 808                         }else{
 809                                 UUID uuid = UUID.randomUUID();
 810                                 String label = row.getCountry();
 811                                 String text = row.getCountry();
 812                                 String labelAbbrev = null;
 813                                 NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
 814                                 NamedAreaLevel level = NamedAreaLevel.COUNTRY();
 815                                 NamedArea newCountry = this.getNamedArea(state, uuid, label, text, labelAbbrev, areaType, level);
 816                                 facade.setCountry(newCountry);
 817                         }
 818                 }
 819         }
 820
 821
 822
 823
 824         protected boolean isInteger(String value){
 825                 try {
 826                         Integer.valueOf(value);
 827                         return true;
 828                 } catch (NumberFormatException e) {
 829                         return false;
 830                 }
 831         }
 832
 833         @Override
 834         protected boolean secondPass(SpecimenCdmExcelImportState state) {
 835                 //no second path defined yet
 836                 return true;
 837         }
 838
 839
 840         @Override
 841         protected String getWorksheetName() {
 842                 return WORKSHEET_NAME;
 843         }
 844
 845         @Override
 846         protected boolean needsNomenclaturalCode() {
 847                 return false;
 848         }
 849
 850
 851         /* (non-Javadoc)
 852          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 853          */
 854         @Override
 855         protected boolean doCheck(SpecimenCdmExcelImportState state) {
 856                 logger.warn("Validation not yet implemented for " + this.getClass().getSimpleName());
 857                 return true;
 858         }
 859
 860
 861
 862         @Override
 863         protected boolean isIgnore(SpecimenCdmExcelImportState state) {
 864                 return !state.getConfig().isDoSpecimen();
 865         }
 866
 867
 868 }