cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/taxa/NormalExplicitImport.java

   1  /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.excel.taxa;
  11
  12 import java.net.MalformedURLException;
  13 import java.net.URI;
  14 import java.net.URISyntaxException;
  15 import java.util.Arrays;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import org.apache.commons.lang.StringUtils;
  24 import org.apache.log4j.Logger;
  25 import org.springframework.stereotype.Component;
  26
  27 import eu.etaxonomy.cdm.common.CdmUtils;
  28 import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;
  29 import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase.SourceDataHolder;
  30 import eu.etaxonomy.cdm.io.tcsrdf.TcsRdfTransformer;
  31 import eu.etaxonomy.cdm.model.agent.Team;
  32 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  33 import eu.etaxonomy.cdm.model.common.CdmBase;
  34 import eu.etaxonomy.cdm.model.common.Extension;
  35 import eu.etaxonomy.cdm.model.common.ExtensionType;
  36 import eu.etaxonomy.cdm.model.common.Language;
  37 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  38 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  39 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  40 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  41 import eu.etaxonomy.cdm.model.description.Distribution;
  42 import eu.etaxonomy.cdm.model.description.Feature;
  43 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  44 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  45 import eu.etaxonomy.cdm.model.description.TaxonNameDescription;
  46 import eu.etaxonomy.cdm.model.description.TextData;
  47 import eu.etaxonomy.cdm.model.location.NamedArea;
  48 import eu.etaxonomy.cdm.model.media.Media;
  49 import eu.etaxonomy.cdm.model.name.INonViralName;
  50 import eu.etaxonomy.cdm.model.name.NameRelationshipType;
  51 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  52 import eu.etaxonomy.cdm.model.name.NomenclaturalStatusType;
  53 import eu.etaxonomy.cdm.model.name.Rank;
  54 import eu.etaxonomy.cdm.model.name.TaxonName;
  55 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  56 import eu.etaxonomy.cdm.model.reference.INomenclaturalReference;
  57 import eu.etaxonomy.cdm.model.reference.Reference;
  58 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  59 import eu.etaxonomy.cdm.model.reference.ReferenceType;
  60 import eu.etaxonomy.cdm.model.taxon.Classification;
  61 import eu.etaxonomy.cdm.model.taxon.Synonym;
  62 import eu.etaxonomy.cdm.model.taxon.SynonymType;
  63 import eu.etaxonomy.cdm.model.taxon.Taxon;
  64 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  65 import eu.etaxonomy.cdm.persistence.dto.MergeResult;
  66 import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
  67 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  68 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  69 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  70
  71 /**
  72  * @author a.babadshanjan
  73  * @created 08.01.2009
  74  */
  75
  76 @Component
  77 public class NormalExplicitImport extends TaxonExcelImporterBase {
  78     private static final long serialVersionUID = 3642423349766191160L;
  79
  80     private static final Logger logger = Logger.getLogger(NormalExplicitImport.class);
  81
  82         public static Set<String> validMarkers = new HashSet<String>(Arrays.asList(new String[]{"", "valid", "accepted", "a", "v", "t", "!"}));
  83         public static Set<String> synonymMarkers = new HashSet<String>(Arrays.asList(new String[]{"**","invalid", "synonym", "s", "i"}));
  84         public static Set<String> nameStatusMarkers = new HashSet<String>(Arrays.asList(new String[]{"illegitimate", "nom. rej.", "nom. cons."}));
  85         public static final UUID uuidRefExtension = UUID.fromString("a46533df-7a78-448f-9b80-36d087fbdf2a");
  86
  87     private static final Object NOM_ILLEG = "illegitimate";
  88     private static final Object NOM_REJ = "nom. rej.";
  89     private static final Object NOM_CONS = "nom. cons.";
  90
  91
  92         /* (non-Javadoc)
  93          * @see eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase#analyzeSingleValue(eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase.KeyValue, eu.etaxonomy.cdm.io.excel.common.ExcelImportState)
  94          */
  95         @Override
  96         protected void analyzeSingleValue(KeyValue keyValue, TaxonExcelImportState state) {
  97
  98                 NormalExplicitRow normalExplicitRow = state.getCurrentRow();
  99                 String key = keyValue.key;
 100                 String value = keyValue.value;
 101                 Integer index = keyValue.index;
 102                 if (((NormalExplicitImportConfigurator)state.getConfig()).getParentUUID() != null){
 103             normalExplicitRow.setParentId("0");
 104                 }
 105
 106                 key = key.replace(" ","");
 107                 if (key.equalsIgnoreCase(ID_COLUMN)) {
 108                         //String ivalue = floatString2IntValue(value);
 109                         normalExplicitRow.setId(value);
 110
 111                 } else if(key.equalsIgnoreCase(PARENT_ID_COLUMN) ) {
 112                         //int ivalue = floatString2IntValue(value);
 113                         normalExplicitRow.setParentId(value);
 114
 115                 } else if(key.equalsIgnoreCase(RANK_COLUMN)) {
 116                         normalExplicitRow.setRank(value);
 117
 118                 } else if(key.trim().equalsIgnoreCase(SCIENTIFIC_NAME_COLUMN) || key.trim().equalsIgnoreCase(FULLNAME_COLUMN)) {
 119                         normalExplicitRow.setScientificName(value);
 120
 121                 } else if(key.equalsIgnoreCase(AUTHOR_COLUMN)|| key.equalsIgnoreCase(AUTHORS_COLUMN)) {
 122                         normalExplicitRow.setAuthor(value);
 123                 }else if(key.equalsIgnoreCase(PUBLISHING_AUTHOR_COLUMN)) {
 124             normalExplicitRow.setPublishingAuthor(value);
 125                 }else if(key.equalsIgnoreCase(BASIONYM_AUTHOR_COLUMN)) {
 126             normalExplicitRow.setBasionymAuthor(value);
 127
 128                 }else if(key.equalsIgnoreCase(BASIONYM_COLUMN)) {
 129             normalExplicitRow.setBasionym(value);
 130                 }else if(key.trim().equalsIgnoreCase(NOMENCLATURAL_SYNONYM_COLUMN)) {
 131             normalExplicitRow.setSynonym(value);
 132                 } else if(key.equalsIgnoreCase(REFERENCE_COLUMN) || key.equalsIgnoreCase(PUBLICATION_COLUMN)) {
 133                         normalExplicitRow.setReference(value);
 134
 135                 } else if(key.equalsIgnoreCase(COLLATION_COLUMN)) {
 136             normalExplicitRow.setCollation(value);
 137
 138         }else if(key.equalsIgnoreCase(NAMESTATUS_COLUMN)) {
 139                         normalExplicitRow.setNameStatus(value);
 140
 141                 } else if(key.equalsIgnoreCase(VERNACULAR_NAME_COLUMN)) {
 142                         normalExplicitRow.setCommonName(value);
 143
 144                 } else if(key.equalsIgnoreCase(LANGUAGE_COLUMN)) {
 145                         normalExplicitRow.setLanguage(value);
 146
 147                 } else if(key.equalsIgnoreCase(TDWG_COLUMN) ) {
 148                         //TODO replace still necessary?
 149                         value = value.replace(".0", "");
 150                         normalExplicitRow.putDistribution(index, value);
 151
 152                 } else if(key.equalsIgnoreCase(PROTOLOGUE_COLUMN)) {
 153                         normalExplicitRow.putProtologue(index, value);
 154
 155                 } else if(key.equalsIgnoreCase(IMAGE_COLUMN)) {
 156                         normalExplicitRow.putImage(index, value);
 157
 158                 } else if(key.equalsIgnoreCase(DATE_COLUMN) || key.equalsIgnoreCase(YEAR_COLUMN)|| key.equalsIgnoreCase(PUBLICATION_YEAR_COLUMN)) {
 159             normalExplicitRow.setDate(value);
 160
 161         } else if(key.equalsIgnoreCase(FAMILY_COLUMN)) {
 162             normalExplicitRow.setFamily(value);
 163         } else if(key.equalsIgnoreCase(INFRA_FAMILY_COLUMN)) {
 164             normalExplicitRow.setInfraFamily(value);
 165         }else if(key.equalsIgnoreCase(GENUS_COLUMN)) {
 166             normalExplicitRow.setGenus(value);
 167         }else if(key.trim().equalsIgnoreCase(INFRA_GENUS_COLUMN.trim())) {
 168             normalExplicitRow.setInfraGenus(value);
 169         }else if(key.equalsIgnoreCase(SPECIES_COLUMN)) {
 170             normalExplicitRow.setSpecies(value);
 171         }else if(key.equalsIgnoreCase(INFRA_SPECIES_COLUMN)) {
 172             normalExplicitRow.setInfraSpecies(value);
 173         } else if (key.equalsIgnoreCase(VERSION_COLUMN)){
 174             normalExplicitRow.setVersion(value);
 175         }
 176
 177
 178
 179
 180         else if(key.equalsIgnoreCase("!")) {
 181             //! = Legitimate, * = Illegitimate, ** = Invalid, *** = nom. rej., !! = nom. cons.
 182             if (value.equals("!")){
 183                 normalExplicitRow.setNameStatus("accepted");
 184             } else if (value.equals("*")){
 185                 normalExplicitRow.setNameStatus("illegitimate");
 186             } else if (value.equals("**")){
 187                 normalExplicitRow.setNameStatus("invalid");
 188             } else if (value.equals("***")){
 189                 normalExplicitRow.setNameStatus("nom. rej.");
 190             } else if (value.equals("!!")){
 191                 normalExplicitRow.setNameStatus("nom. cons.");
 192             } else{
 193                 normalExplicitRow.setNameStatus("accepted");
 194             }
 195         }else {
 196                         if (analyzeFeatures(state, keyValue)){
 197
 198                         }else{
 199                                 String message = "Unexpected column header " + key;
 200                                 fireWarningEvent(message, state, 10);
 201                                 state.setUnsuccessfull();
 202                                 //logger.error(message);
 203                         }
 204                 }
 205                 return;
 206         }
 207
 208
 209         /**
 210          *  Create base taxa and add all information attached to it's name.
 211          */
 212         @Override
 213     protected void firstPass(TaxonExcelImportState state) {
 214
 215 //              if (1==1){
 216 //                      return;
 217 //              }
 218 //              System.out.println("FP:" + state.getCurrentLine());
 219                 Rank rank = null;
 220                 NormalExplicitRow taxonDataHolder = state.getCurrentRow();
 221
 222                 String rankStr = taxonDataHolder.getRank();
 223                 String taxonNameStr = taxonDataHolder.getScientificName();
 224                 String authorStr = taxonDataHolder.getAuthor();
 225                 String publishingAuthor= taxonDataHolder.getPublishingAuthor();
 226                 String basionymAuthor = taxonDataHolder.getBasionymAuthor();
 227
 228                 String referenceStr = taxonDataHolder.getReference();
 229                 String nameStatus = taxonDataHolder.getNameStatus();
 230                 String familyNameStr = taxonDataHolder.getFamily();
 231                 String infraFamilyNameStr = taxonDataHolder.getInfraFamily();
 232                 String genusNameStr = taxonDataHolder.getGenus();
 233                 String infraGenusNameStr = taxonDataHolder.getInfraGenus();
 234                 String speciesNameStr = taxonDataHolder.getSpecies();
 235                 String infraSpeciesNameStr = taxonDataHolder.getInfraSpecies();
 236
 237                 String version = taxonDataHolder.getVersion();
 238
 239
 240
 241
 242                 String dateStr = taxonDataHolder.getDate();
 243                 String id = taxonDataHolder.getId();
 244                 UUID cdmUuid = taxonDataHolder.getCdmUuid();
 245
 246                 TaxonBase<?> taxonBase = null;
 247                 if (cdmUuid != null){
 248                         taxonBase = getTaxonService().find(cdmUuid);
 249                 }else{
 250                         if (StringUtils.isNotBlank(taxonNameStr)) {
 251
 252                                 // Rank
 253                                 try {
 254                                     if (!StringUtils.isBlank(rankStr)) {
 255                         rank = Rank.getRankByNameOrIdInVoc(rankStr);
 256                         }
 257                                 } catch (UnknownCdmTypeException ex) {
 258                                         try {
 259                                                 rank = Rank.getRankByEnglishName(rankStr, state.getConfig().getNomenclaturalCode(), false);
 260                                         } catch (UnknownCdmTypeException e) {
 261                                             try {
 262                                    rank = TcsRdfTransformer.rankString2Rank(rankStr);
 263                                } catch (UnknownCdmTypeException e1) {
 264                                     // TODO Auto-generated catch block
 265                                     e1.printStackTrace();
 266                                }
 267
 268                                         }
 269                                 }
 270
 271                     //taxon
 272                                 taxonBase = createTaxon(state, rank, taxonNameStr, authorStr, publishingAuthor, basionymAuthor, referenceStr, dateStr, nameStatus);
 273                         }else{
 274                                 return;
 275                         }
 276                 }
 277                 if (taxonBase == null){
 278                         String message = "Taxon is already in DB. Record will not be handled";
 279                         fireWarningEvent(message, "Record: " + state.getCurrentLine(), 6);
 280                         logger.warn(message);
 281                         //state.setUnsuccessfull();
 282                         return;
 283                 }
 284
 285                 //protologue
 286                 for (String protologue : taxonDataHolder.getProtologues()){
 287                         TextData textData = TextData.NewInstance(Feature.PROTOLOGUE());
 288                         this.getNameDescription(taxonBase.getName(), state).addElement(textData);
 289                         URI uri;
 290                         try {
 291                                 uri = new URI(protologue);
 292                                 textData.addMedia(Media.NewInstance(uri, null, null, null));
 293
 294                         } catch (URISyntaxException e) {
 295                                 String warning = "URISyntaxException when trying to convert to URI: " + protologue;
 296                                 logger.error(warning);
 297                                 state.setUnsuccessfull();
 298                         }
 299                 }
 300
 301                 //state.putTaxon(id, taxonBase);
 302                 taxonBase = getTaxonService().save(taxonBase);
 303                 taxonDataHolder.setCdmUuid(taxonBase.getUuid());
 304                 if (id == "0"){
 305                     state.putTaxon(taxonNameStr, taxonBase);
 306                 }else{
 307                     state.putTaxon(id, taxonBase);
 308                 }
 309                 return;
 310     }
 311
 312
 313
 314         /**
 315          *  Stores parent-child, synonym and common name relationships.
 316          *  Adds all taxon related descriptive information (this is not done in the first pass
 317          *  because the information may also be attached to a synonym).
 318          */
 319         @Override
 320     protected void secondPass(TaxonExcelImportState state) {
 321                 if (logger.isDebugEnabled()){logger.debug(state.getCurrentLine());}
 322                 try {
 323                         NormalExplicitRow taxonDataHolder = state.getCurrentRow();
 324                         String taxonNameStr = taxonDataHolder.getScientificName();
 325                         String nameStatus = taxonDataHolder.getNameStatus();
 326                         String commonNameStr = taxonDataHolder.getCommonName();
 327
 328                         String synonymNameStr = taxonDataHolder.getSynonym();
 329                         String basionymNameStr = taxonDataHolder.getBasionym();
 330
 331                         String parentId = taxonDataHolder.getParentId();
 332                         String childId = taxonDataHolder.getId();
 333                         UUID cdmUuid = taxonDataHolder.getCdmUuid();
 334                         Taxon acceptedTaxon = null;
 335                         TaxonName nameUsedInSource = null;
 336                         TaxonBase<?> taxonBase = null;
 337                         Taxon parentTaxon = null;
 338
 339                         if (cdmUuid != null){
 340                                 taxonBase = getTaxonService().find(cdmUuid);
 341                                 if (taxonBase != null ){
 342                     acceptedTaxon = getAcceptedTaxon(taxonBase);
 343                     nameUsedInSource = taxonBase.getName();
 344                                 }
 345                         } else{
 346                             taxonBase = state.getTaxonBase(childId);
 347
 348                          if (parentId == "0" && state.getParent() == null){
 349                      parentTaxon =(Taxon) getTaxonService().load(((NormalExplicitImportConfigurator)state.getConfig()).getParentUUID());
 350                      state.setParent(parentTaxon);
 351                  }else if (parentId != "0"){
 352                     parentTaxon = CdmBase.deproxy(state.getTaxonBase(parentId), Taxon.class);
 353                  } else if (state.getParent() != null){
 354                      parentTaxon = state.getParent();
 355                  }
 356                         if (taxonBase != null ){
 357                                 acceptedTaxon = getAcceptedTaxon(taxonBase);
 358                                 if (synonymNameStr != null){
 359                                     Synonym syn = createSynonym(state,taxonBase,synonymNameStr);
 360                                     acceptedTaxon.addSynonym(syn, SynonymType.HETEROTYPIC_SYNONYM_OF());
 361                                 }
 362                                 if (basionymNameStr != null){
 363                                     Synonym syn = createSynonym(state,taxonBase,basionymNameStr);
 364                         acceptedTaxon.addSynonym(syn, SynonymType.HOMOTYPIC_SYNONYM_OF());
 365                         syn.getName().addRelationshipToName(acceptedTaxon.getName(), NameRelationshipType.BASIONYM(), null);
 366                                 }
 367                                 nameUsedInSource = taxonBase.getName();
 368
 369                             //TODO error handling for class cast
 370
 371
 372                                 nameUsedInSource = taxonBase.getName();
 373                                 nameStatus = CdmUtils.Nz(nameStatus).trim().toLowerCase();
 374                                 if (validMarkers.contains(nameStatus)){
 375                                                 Taxon taxon = CdmBase.deproxy(taxonBase, Taxon.class);
 376                                                 acceptedTaxon = taxon;
 377                                                 // Add the parent relationship
 378                                                 //if (state.getCurrentRow().getParentId() != 0) {
 379                                                 MergeResult result = null;
 380                                                         if (parentTaxon != null) {
 381                                                                 //Taxon taxon = (Taxon)state.getTaxonBase(childId);
 382
 383                                                         //      Reference sourceRef = state.getConfig().getSourceReference();
 384                                                                 String microCitation = null;
 385                                                                 Taxon childTaxon = taxon;
 386                                                                 makeParent(state, parentTaxon, childTaxon, null, microCitation);
 387                                                                 getTaxonService().saveOrUpdate(childTaxon);
 388                                                                 state.putTaxon(parentId, parentTaxon);
 389                                                         } else {
 390                                                                 String message = "Taxonomic parent not found for " + taxonNameStr;
 391                                                                 logger.warn(message);
 392                                                                 fireWarningEvent(message, state, 6);
 393                                                                 //state.setUnsuccessfull();
 394                                                         }
 395     //                                          }else{
 396     //                                                  //do nothing (parent == 0) no parent exists
 397     //                                          }
 398                                         }else if (synonymMarkers.contains(nameStatus)){
 399                                                 //add synonym relationship
 400                                                 acceptedTaxon = parentTaxon;
 401                                                 try {
 402                                                         Synonym synonym = CdmBase.deproxy(taxonBase,Synonym.class);
 403                                                         if (acceptedTaxon == null){
 404                                                                 String message = "Accepted/valid taxon could not be found. Please check referential integrity.";
 405                                                                 fireWarningEvent(message, state, 8);
 406                                                         }else{
 407                                                             if (parentId != "0"){
 408                                                                 //if no relation was defined in file skip relationship creation
 409                                                                 acceptedTaxon.addSynonym(synonym, SynonymType.SYNONYM_OF());
 410                                                                 getTaxonService().saveOrUpdate(acceptedTaxon);
 411                                                             }
 412                                                         }
 413                                                 } catch (Exception e) {
 414                                                         String message = "Unhandled exception (%s) occurred during synonym import/update";
 415                                                         message = String.format(message, e.getMessage());
 416                                                         fireWarningEvent(message, state, 10);
 417                                                         state.setUnsuccessfull();
 418                                                 }
 419
 420                                         }else{
 421                                             if (parentTaxon != null) {
 422                                 Taxon taxon = (Taxon)state.getTaxonBase(childId);
 423
 424                                // Reference sourceRef = state.getConfig().getSourceReference();
 425                                 String microCitation = null;
 426                                 Taxon childTaxon = taxon;
 427                                 makeParent(state, parentTaxon, childTaxon, null, microCitation);
 428                                 getTaxonService().saveOrUpdate(parentTaxon);
 429                                 state.putTaxon(parentId, parentTaxon);
 430                             } else {
 431                                 String message = "Taxonomic parent not found for " + taxonNameStr;
 432                                 logger.warn(message);
 433                                 fireWarningEvent(message, state, 6);
 434                                 //state.setUnsuccessfull();
 435                             }
 436                                         }
 437
 438
 439                         }
 440                         if (StringUtils.isBlank(taxonNameStr) && acceptedTaxon == null) {
 441                     acceptedTaxon = parentTaxon;
 442                     nameUsedInSource = null;
 443                 }
 444                         }
 445                         if (acceptedTaxon == null && (StringUtils.isNotBlank(commonNameStr) ||taxonDataHolder.getFeatures().size() > 0 )){
 446                                 String message = "Accepted taxon could not be found. Can't add additional data (common names, descriptive data, ...) to taxon";
 447                                 fireWarningEvent(message, state, 6);
 448                         }else{
 449                                 //common names
 450                                 if (StringUtils.isNotBlank(commonNameStr)){                     // add common name to taxon
 451                                         handleCommonName(state, taxonNameStr, commonNameStr, acceptedTaxon);
 452                                 }
 453
 454
 455                                 //media
 456                                 for (String imageUrl : taxonDataHolder.getImages()){
 457                                         TaxonDescription td = acceptedTaxon.getImageGallery(true);
 458                                         DescriptionElementBase mediaHolder;
 459                                         if (td.getElements().size() != 0){
 460                                                 mediaHolder = td.getElements().iterator().next();
 461                                         }else{
 462                                                 mediaHolder = TextData.NewInstance(Feature.IMAGE());
 463                                                 td.addElement(mediaHolder);
 464                                         }
 465                                         try {
 466                                                 Media media = getImageMedia(imageUrl, READ_MEDIA_DATA);
 467                                                 mediaHolder.addMedia(media);
 468                                         } catch (MalformedURLException e) {
 469                                                 logger.warn("Can't add media: " + e.getMessage());
 470                                                 state.setUnsuccessfull();
 471                                         }
 472                                 }
 473
 474                                 //tdwg label
 475                                 for (String tdwg : taxonDataHolder.getDistributions()){
 476                                         TaxonDescription td = this.getTaxonDescription(acceptedTaxon, state.getConfig().getSourceReference() ,false, true);
 477                                         NamedArea area = TdwgAreaProvider.getAreaByTdwgAbbreviation(tdwg);
 478                                         if (area == null){
 479                                                 area = TdwgAreaProvider.getAreaByTdwgLabel(tdwg);
 480                                         }
 481                                         if (area != null){
 482                                                 Distribution distribution = Distribution.NewInstance(area, PresenceAbsenceTerm.PRESENT());
 483                                                 td.addElement(distribution);
 484                                         }else{
 485                                                 String message = "TDWG area could not be recognized: " + tdwg;
 486                                                 logger.warn(message);
 487                                                 state.setUnsuccessfull();
 488                                         }
 489                                 }
 490
 491                                 //features
 492                                 handleFeatures(state, taxonDataHolder, acceptedTaxon, nameUsedInSource);
 493                         }
 494                 } catch (Exception e) {
 495                         e.printStackTrace();
 496                 }
 497                 return;
 498         }
 499
 500
 501         /**
 502      * @param state
 503      * @param taxonBase
 504      * @param synonymNameStr
 505      */
 506     private Synonym createSynonym(TaxonExcelImportState state, TaxonBase<?> taxonBase, String synonymNameStr) {
 507         NomenclaturalCode nc = state.getConfig().getNomenclaturalCode();
 508         TaxonName name = null;
 509         if (nc.isKindOf(NomenclaturalCode.ICZN)){
 510             name = TaxonNameFactory.NewZoologicalInstance(taxonBase.getName().getRank());
 511         }else if (nc.isKindOf(NomenclaturalCode.ICNAFP)){
 512             name = TaxonNameFactory.NewBotanicalInstance(taxonBase.getName().getRank());
 513         } else{
 514             name = TaxonNameFactory.NewNonViralInstance(taxonBase.getName().getRank());
 515         }
 516         name.setTitleCache(synonymNameStr, true);
 517         if (name != null){
 518             return Synonym.NewInstance(name, null);
 519         }
 520         logger.debug("The nomenclatural code is not supported.");
 521         return null;
 522     }
 523
 524
 525     /**
 526          * @param state
 527          * @param taxonDataHolder
 528          * @param acceptedTaxon
 529          */
 530         private void handleFeatures(TaxonExcelImportState state, NormalExplicitRow taxonDataHolder, Taxon acceptedTaxon, TaxonName nameUsedInSource) {
 531                 //feature
 532                 for (UUID featureUuid : taxonDataHolder.getFeatures()){
 533                         Feature feature = getFeature(state, featureUuid);
 534                         List<String> textList = taxonDataHolder.getFeatureTexts(featureUuid);
 535                         List<String> languageList = taxonDataHolder.getFeatureLanguages(featureUuid);
 536
 537                         for (int i = 0; i < textList.size(); i++){
 538                                 String featureText = textList.get(i);
 539                                 String featureLanguage = languageList == null ? null :languageList.get(i);
 540                                 Language language = getFeatureLanguage(featureLanguage, state);
 541                                 //TODO
 542                                 TaxonDescription td = this.getTaxonDescription(acceptedTaxon, state.getConfig().getSourceReference() ,false, true);
 543                                 TextData textData = TextData.NewInstance(feature);
 544                                 textData.putText(language, featureText);
 545                                 td.addElement(textData);
 546
 547                                 SourceDataHolder sourceDataHolder = taxonDataHolder.getFeatureTextReferences(featureUuid, i);
 548                                 List<Map<SourceType, String>> sourceList = sourceDataHolder.getSources();
 549                                 for (Map<SourceType, String> sourceMap : sourceList){
 550
 551                                         //ref
 552                                         Reference ref = ReferenceFactory.newGeneric();
 553                                         boolean refExists = false; //in case none of the ref fields exists, the ref should not be added
 554                                         for (SourceType type : sourceMap.keySet()){
 555                                                 String value = sourceMap.get(type);
 556                                                 if (type.equals(SourceType.Author)){
 557                                                         TeamOrPersonBase<?> author = getAuthorAccordingToConfig(value, state);
 558                                                         ref.setAuthorship(author);
 559                                                 }else if (type.equals(SourceType.Title)) {
 560                                                         ref.setTitle(value);
 561                                                 }else if (type.equals(SourceType.Year)) {
 562                                                         ref.setDatePublished(TimePeriodParser.parseString(value));
 563                                                 }else if (type.equals(SourceType.RefExtension)) {
 564                                                         ExtensionType extensionType = getExtensionType(state, uuidRefExtension, "RefExtension", "Reference Extension", "RefExt.");
 565                                                         Extension extension = Extension.NewInstance(ref, value, extensionType);
 566                                                 }
 567                                                 refExists = true;
 568                                         }
 569                                         DescriptionElementSource source = DescriptionElementSource.NewInstance(OriginalSourceType.PrimaryTaxonomicSource);
 570                                         if (refExists){
 571                                                 ref = getReferenceAccordingToConfig(ref, state);
 572                                                 source.setCitation(ref);
 573                                                 source.setNameUsedInSource(nameUsedInSource);
 574                                         }
 575                                         textData.addSource(source);
 576                                 }
 577                         }
 578                 }
 579         }
 580
 581         private final Map<String, UUID> referenceMapping = new HashMap<String, UUID>();
 582         private final Map<UUID, Reference> referenceStore = new HashMap<UUID, Reference>();
 583
 584         private Reference getReferenceAccordingToConfig(Reference value, TaxonExcelImportState state) {
 585                 Reference result = null;
 586                 String titleCache = value.getTitleCache();
 587                 UUID referenceUuid = referenceMapping.get(titleCache);
 588                 if (referenceUuid != null){
 589                         result = referenceStore.get(referenceUuid);
 590                 }
 591                 if (result == null){
 592                         result = value;
 593                         referenceStore.put(result.getUuid(), result);
 594                 }
 595                 if (referenceUuid == null){
 596                         referenceMapping.put(titleCache, result.getUuid());
 597                 }
 598                 return result;
 599         }
 600
 601
 602         private final Map<String, UUID> authorMapping = new HashMap<String, UUID>();
 603         private final Map<UUID, TeamOrPersonBase> authorStore = new HashMap<UUID, TeamOrPersonBase>();
 604
 605         private TeamOrPersonBase<?> getAuthorAccordingToConfig(String value, TaxonExcelImportState state) {
 606                 TeamOrPersonBase<?> result = null;
 607                 UUID authorUuid = authorMapping.get(value);
 608                 if (authorUuid != null){
 609                         result = authorStore.get(authorUuid);
 610                 }
 611                 if (result == null){
 612                         //TODO parsing
 613                         TeamOrPersonBase<?> author = Team.NewInstance();
 614                         author.setTitleCache(value, true);
 615                         result = author;
 616                         authorStore.put(result.getUuid(), result);
 617                 }
 618                 if (authorUuid == null){
 619                         authorMapping.put(value, result.getUuid());
 620                 }
 621                 return result;
 622         }
 623
 624
 625         private final Map<String, UUID> languageMapping = new HashMap<String, UUID>();
 626
 627         private Language getFeatureLanguage(String featureLanguage, TaxonExcelImportState state) {
 628                 if (StringUtils.isBlank(featureLanguage)){
 629                         return null;
 630                 }
 631                 UUID languageUuid = languageMapping.get(featureLanguage);
 632                 if (languageUuid == null){
 633                         Language result = getTermService().getLanguageByIso(featureLanguage);
 634                         languageUuid = result.getUuid();
 635                         languageMapping.put(featureLanguage, languageUuid);
 636                 }
 637                 Language result = getLanguage(state, languageUuid, null, null, null);
 638                 return result;
 639         }
 640
 641
 642         /**
 643          * @param state
 644          * @param taxonNameStr
 645          * @param commonNameStr
 646          * @param parentId
 647          */
 648         private void handleCommonName(TaxonExcelImportState state,
 649                         String taxonNameStr, String commonNameStr, Taxon acceptedTaxon) {
 650                 Language language = getTermService().getLanguageByIso(state.getCurrentRow().getLanguage());
 651                 if (language == null && CdmUtils.isNotEmpty(state.getCurrentRow().getLanguage())  ){
 652                         String error ="Language is null but shouldn't";
 653                         logger.error(error);
 654                         throw new IllegalArgumentException(error);
 655                 }
 656                 CommonTaxonName commonTaxonName = CommonTaxonName.NewInstance(commonNameStr, language);
 657                 try {
 658                         TaxonDescription taxonDescription = getTaxonDescription(acceptedTaxon, false, true);
 659                         taxonDescription.addElement(commonTaxonName);
 660                         logger.info("Common name " + commonNameStr + " added to " + acceptedTaxon.getTitleCache());
 661                 } catch (ClassCastException ex) {
 662                         logger.error(taxonNameStr + " is not a taxon instance.");
 663                 }
 664         }
 665
 666
 667         /**
 668          * @param state
 669          * @param rank
 670          * @param taxonNameStr
 671          * @param authorStr
 672          * @param nameStatus
 673          * @param nameStatus2
 674          * @return
 675          */
 676         private TaxonBase createTaxon(TaxonExcelImportState state, Rank rank,
 677                 String familyNameStr, String infraFamilyNameStr, String genusNameStr, String infraGenusNameStr, String speciesNameStr, String infraSpeciesNameStr, String authorStr, String publishingAuthorStr, String basionymAuthorStr,String reference, String date, String nameStatus) {
 678                 // Create the taxon name object depending on the setting of the nomenclatural code
 679                 // in the configurator (botanical code, zoological code, etc.)
 680
 681                 NomenclaturalCode nc = getConfigurator().getNomenclaturalCode();
 682
 683                 TaxonBase taxonBase;
 684                 String nameCache = null;
 685                 if (rank == null){
 686                     System.err.println("bla");
 687                 }
 688                 if (rank.isGenus()){
 689                     nameCache =genusNameStr;
 690                 } else if (rank.isInfraGeneric()){
 691                     nameCache =CdmUtils.concat(" " +rank.getIdInVocabulary() + " ",genusNameStr,infraGenusNameStr);
 692
 693         } else if (rank.isSpecies()){
 694             nameCache = CdmUtils.concat(" ", genusNameStr,speciesNameStr);
 695         } else if (rank.isInfraSpecific()){
 696             nameCache = CdmUtils.concat(" " +rank.getIdInVocabulary() + " ",genusNameStr,infraGenusNameStr);
 697         }
 698                 if (! synonymMarkers.contains(nameStatus)  && state.getConfig().isReuseExistingTaxaWhenPossible()){
 699                         taxonBase = getTaxonService().findBestMatchingTaxon(nameCache);
 700                 }else{
 701                         taxonBase = getTaxonService().findBestMatchingSynonym(nameCache);
 702                         if (taxonBase != null){
 703                                 logger.info("Matching taxon/synonym found for " + nameCache);
 704                         }
 705                 }
 706                 if (taxonBase != null){
 707                         logger.info("Matching taxon/synonym found for " + nameCache);
 708                         return null;
 709                 }else {
 710                         taxonBase = createTaxon(state, rank, nameCache, authorStr, publishingAuthorStr, basionymAuthorStr, reference, date, nameStatus, nc);
 711                 }
 712                 return taxonBase;
 713         }
 714
 715         /**
 716      * @param state
 717      * @param rank
 718      * @param taxonNameStr
 719      * @param authorStr
 720      * @param nameStatus
 721      * @param nameStatus2
 722      * @return
 723      */
 724     private TaxonBase createTaxon(TaxonExcelImportState state, Rank rank,
 725             String taxonNameStr, String authorStr, String publishingAuthorStr, String basionymAuthorStr,String reference, String date, String nameStatus) {
 726         // Create the taxon name object depending on the setting of the nomenclatural code
 727         // in the configurator (botanical code, zoological code, etc.)
 728         if (StringUtils.isBlank(taxonNameStr)){
 729             return null;
 730         }
 731         NomenclaturalCode nc = getConfigurator().getNomenclaturalCode();
 732
 733         TaxonBase taxonBase = null;
 734
 735         String titleCache = CdmUtils.concat(" ", taxonNameStr, authorStr);
 736         if (! synonymMarkers.contains(nameStatus)  && state.getConfig().isReuseExistingTaxaWhenPossible()){
 737             titleCache = CdmUtils.concat(" ", taxonNameStr, authorStr);
 738             taxonBase = getTaxonService().findBestMatchingTaxon(titleCache);
 739         }else if ( state.getConfig().isReuseExistingTaxaWhenPossible()){
 740             taxonBase = getTaxonService().findBestMatchingSynonym(titleCache);
 741             if (taxonBase != null){
 742                 logger.info("Matching taxon/synonym found for " + titleCache);
 743             }
 744         }
 745         if (taxonBase != null && taxonBase.getName().getTitleCache().equals(CdmUtils.concat(" ", taxonNameStr, authorStr))){
 746             logger.info("Matching taxon/synonym found for " + titleCache + " - "+taxonBase.getTitleCache());
 747             return null;
 748         }else {
 749             taxonBase = createTaxon(state, rank, taxonNameStr, authorStr, publishingAuthorStr, basionymAuthorStr, reference, date, nameStatus, nc);
 750         }
 751         return taxonBase;
 752     }
 753
 754
 755
 756         /**
 757          * @param state
 758          * @param rank
 759          * @param taxonNameStr
 760          * @param authorStr
 761          * @param nameStatus
 762          * @param nameStatus2
 763          * @param nc
 764          * @return
 765          */
 766         private TaxonBase<?> createTaxon(TaxonExcelImportState state, Rank rank, String taxonNameStr,
 767                         String authorStr, String publishingAutorStr, String basionymAuthor, String reference, String date, String nameStatus, NomenclaturalCode nc) {
 768                 TaxonBase<?> taxonBase;
 769                 INonViralName taxonName = null;
 770                 if (nc == NomenclaturalCode.ICVCN){
 771                         logger.warn("ICVCN not yet supported");
 772
 773                 }else{
 774                     //String taxonNameStr = titleCache.substring(0, titleCache.indexOf(authorStr));
 775                         taxonName = nc.getNewTaxonNameInstance(rank);
 776                         NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
 777                         taxonName = parser.parseFullName(taxonNameStr, nc, rank);
 778
 779                         if (! taxonName.getNameCache().equals(taxonNameStr)){
 780                                 taxonName.setNameCache(taxonNameStr, true);
 781                         }
 782
 783                         // Create the author
 784                         if (StringUtils.isNotBlank(authorStr)) {
 785                                 try {
 786                                         parser.parseAuthors(taxonName, authorStr);
 787                                 } catch (StringNotParsableException e) {
 788                                         taxonName.setAuthorshipCache(authorStr);
 789                                 }
 790                         }
 791                         if (StringUtils.isNotBlank(reference)) {
 792                             String pub = CdmUtils.concat(" ", reference, state.getCurrentRow().getCollation());
 793                             String[] split = pub.split(":");
 794                             pub = split[0];
 795                             INomenclaturalReference ref = parser.parseReferenceTitle(pub, date, true);
 796                             if (split.length > 1){
 797                     String detail = split[split.length-1];
 798                     taxonName.setNomenclaturalMicroReference(detail.trim());
 799
 800                 }
 801
 802              //   ref.setAbbrevTitle(pub);
 803
 804
 805                             if (ref.getAuthorship() == null){
 806                                 ref.setAuthorship(taxonName.getCombinationAuthorship());
 807                             }
 808
 809                             if (ref.getAbbrevTitle() == null && !ref.isOfType(ReferenceType.Article)) {
 810                     ref.setAbbrevTitle(reference);
 811                     ref.setProtectedAbbrevTitleCache(false);
 812                 }
 813
 814                 ref.setProtectedTitleCache(false);
 815                 taxonName.setNomenclaturalReference(ref);
 816                           //  taxonName.setNomenclaturalMicroReference(state.getCurrentRow().getCollation());
 817                         }
 818                 }
 819
 820                 //Create the taxon
 821                 //Reference sec = state.getConfig().getSourceReference();
 822                 // Create the status
 823                 nameStatus = CdmUtils.Nz(nameStatus).trim().toLowerCase();
 824                 if (validMarkers.contains(nameStatus)){
 825                         taxonBase = Taxon.NewInstance(taxonName, null);
 826                 }else if (synonymMarkers.contains(nameStatus)){
 827                         taxonBase = Synonym.NewInstance(taxonName, null);
 828                 }else {
 829                         Taxon taxon = Taxon.NewInstance(taxonName, null);
 830                         if (nameStatusMarkers.contains(nameStatus)){
 831                             if (nameStatus.equals(NOM_ILLEG)){
 832                                 taxonName.addStatus(NomenclaturalStatusType.ILLEGITIMATE(), null, null);
 833                             } else if (nameStatus.equals(NOM_REJ)){
 834                                 taxonName.addStatus(NomenclaturalStatusType.REJECTED(), null, null);
 835                             } else if (nameStatus.equals(NOM_CONS)){
 836                     taxonName.addStatus(NomenclaturalStatusType.CONSERVED(), null, null);
 837                 }
 838                         }else{
 839                             taxon.setTaxonStatusUnknown(true);
 840                         }
 841                         taxonBase = taxon;
 842                 }
 843                 taxonBase.getName().addSource(OriginalSourceType.Import, null,"TaxonName" ,state.getConfig().getSourceReference(), null);
 844                 taxonBase.addSource(OriginalSourceType.Import, null,"TaxonName" ,state.getConfig().getSourceReference(), null);
 845
 846                 return taxonBase;
 847         }
 848
 849         /**
 850          * @param taxon
 851          * @return
 852          */
 853         //TODO implementation must be improved when matching of taxon names with existing names is implemented
 854         //=> the assumption that the only description is the description added by this import
 855         //is wrong then
 856         private TaxonNameDescription getNameDescription(TaxonName name, TaxonExcelImportState state) {
 857                 Set<TaxonNameDescription> descriptions = name.getDescriptions();
 858                 if (descriptions.size()>1){
 859                         throw new IllegalStateException("Implementation does not yet support names with multiple descriptions");
 860                 }else if (descriptions.size()==1){
 861                         return descriptions.iterator().next();
 862                 }else{
 863                     TaxonNameDescription desc = TaxonNameDescription.NewInstance(name);
 864                     desc.addSource(OriginalSourceType.Import, null, "NameDescription", state.getConfig().getSourceReference(), null);
 865                         return desc;
 866                 }
 867         }
 868
 869         private void makeParent(TaxonExcelImportState state, Taxon parentTaxon, Taxon childTaxon, Reference citation, String microCitation){
 870                 Reference sec = state.getConfig().getSourceReference();
 871
 872 //              Reference sec = parentTaxon.getSec();
 873                 Classification tree = state.getClassification();
 874                 if (tree == null){
 875                         //tree = makeTree(state, sec);
 876                     if (state.getConfig().getClassificationUuid() != null){
 877                         tree = getClassificationService().load(state.getConfig().getClassificationUuid());
 878                         state.setClassification(tree);
 879                     }
 880                     if (tree == null){
 881                         tree = makeTree(state, sec);
 882                         getClassificationService().save(tree);
 883                         state.setClassification(tree);
 884                     }
 885                 }
 886                 //if (sec.equals(childTaxon.getSec())){
 887                     boolean success =  (null !=  tree.addParentChild(parentTaxon, childTaxon, citation, microCitation));
 888                         if (success == false){
 889                                 state.setUnsuccessfull();
 890                         }
 891 //              }else{
 892 //                      logger.warn("No relationship added for child " + childTaxon.getTitleCache());
 893 //              }
 894                 return;
 895         }
 896
 897
 898         /* (non-Javadoc)
 899          * @see eu.etaxonomy.cdm.io.excel.common.ExcelTaxonOrSpecimenImportBase#createDataHolderRow()
 900          */
 901         @Override
 902         protected NormalExplicitRow createDataHolderRow() {
 903                 return new NormalExplicitRow();
 904         }
 905
 906
 907
 908         /* (non-Javadoc)
 909          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 910          */
 911         @Override
 912         protected boolean doCheck(TaxonExcelImportState state) {
 913                 logger.warn("DoCheck not yet implemented for NormalExplicitImport");
 914                 return true;
 915         }
 916
 917         /* (non-Javadoc)
 918          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IoStateBase)
 919          */
 920         @Override
 921         protected boolean isIgnore(TaxonExcelImportState state) {
 922                 return false;
 923         }
 924
 925
 926
 927 }