cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/wfo/out/WfoContentExport.java

   1 /**
   2 * Copyright (C) 2017 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.io.wfo.out;
  10
  11 import java.io.File;
  12 import java.util.ArrayList;
  13 import java.util.HashSet;
  14 import java.util.Iterator;
  15 import java.util.List;
  16 import java.util.Map;
  17 import java.util.Set;
  18 import java.util.stream.Collectors;
  19 import java.util.stream.Stream;
  20
  21 import org.apache.commons.lang3.StringUtils;
  22 import org.joda.time.DateTime;
  23 import org.joda.time.DateTimeFieldType;
  24 import org.joda.time.Partial;
  25 import org.joda.time.format.DateTimeFormatter;
  26 import org.joda.time.format.DateTimeFormatterBuilder;
  27 import org.springframework.stereotype.Component;
  28
  29 import eu.etaxonomy.cdm.common.CdmUtils;
  30 import eu.etaxonomy.cdm.common.SetMap;
  31 import eu.etaxonomy.cdm.common.URI;
  32 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  33 import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
  34 import eu.etaxonomy.cdm.format.description.CategoricalDataFormatter;
  35 import eu.etaxonomy.cdm.format.reference.NomenclaturalSourceFormatter;
  36 import eu.etaxonomy.cdm.io.common.CdmExportBase;
  37 import eu.etaxonomy.cdm.io.common.ExportResult.ExportResultState;
  38 import eu.etaxonomy.cdm.io.common.TaxonNodeOutStreamPartitioner;
  39 import eu.etaxonomy.cdm.io.common.XmlExportState;
  40 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  41 import eu.etaxonomy.cdm.io.common.mapping.out.IExportTransformer;
  42 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  43 import eu.etaxonomy.cdm.model.common.Annotation;
  44 import eu.etaxonomy.cdm.model.common.AnnotationType;
  45 import eu.etaxonomy.cdm.model.common.CdmBase;
  46 import eu.etaxonomy.cdm.model.common.ICdmBase;
  47 import eu.etaxonomy.cdm.model.common.IIdentifiableEntity;
  48 import eu.etaxonomy.cdm.model.common.Identifier;
  49 import eu.etaxonomy.cdm.model.common.Language;
  50 import eu.etaxonomy.cdm.model.common.LanguageString;
  51 import eu.etaxonomy.cdm.model.common.TimePeriod;
  52 import eu.etaxonomy.cdm.model.description.CategoricalData;
  53 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  54 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  55 import eu.etaxonomy.cdm.model.description.Distribution;
  56 import eu.etaxonomy.cdm.model.description.Feature;
  57 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  58 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  59 import eu.etaxonomy.cdm.model.description.TextData;
  60 import eu.etaxonomy.cdm.model.location.Country;
  61 import eu.etaxonomy.cdm.model.location.NamedArea;
  62 import eu.etaxonomy.cdm.model.name.HomotypicalGroup;
  63 import eu.etaxonomy.cdm.model.name.Rank;
  64 import eu.etaxonomy.cdm.model.name.TaxonName;
  65 import eu.etaxonomy.cdm.model.reference.Reference;
  66 import eu.etaxonomy.cdm.model.taxon.Classification;
  67 import eu.etaxonomy.cdm.model.taxon.Synonym;
  68 import eu.etaxonomy.cdm.model.taxon.Taxon;
  69 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  70 import eu.etaxonomy.cdm.model.term.IdentifierType;
  71
  72 /**
  73  * Classification or taxon tree exporter into WFO Content format.
  74  *
  75  * @author a.mueller
  76  * @since 2024-01-30
  77  */
  78 @Component
  79 public class WfoContentExport
  80         extends CdmExportBase<WfoContentExportConfigurator,WfoContentExportState,IExportTransformer,File>{
  81
  82     private static final long serialVersionUID = -4560488499411723333L;
  83
  84     public WfoContentExport() {
  85         this.ioName = this.getClass().getSimpleName();
  86     }
  87
  88     @Override
  89     public long countSteps(WfoContentExportState state) {
  90         TaxonNodeFilter filter = state.getConfig().getTaxonNodeFilter();
  91         return getTaxonNodeService().count(filter);
  92     }
  93
  94     @Override
  95     protected void doInvoke(WfoContentExportState state) {
  96
  97         try {
  98             IProgressMonitor monitor = state.getConfig().getProgressMonitor();
  99             WfoContentExportConfigurator config = state.getConfig();
 100
 101             //set root node
 102             if (config.getTaxonNodeFilter().hasClassificationFilter()) {
 103                 Classification classification = getClassificationService()
 104                         .load(config.getTaxonNodeFilter().getClassificationFilter().get(0).getUuid());
 105                 state.setRootId(classification.getRootNode().getUuid());
 106             } else if (config.getTaxonNodeFilter().hasSubtreeFilter()) {
 107                 state.setRootId(config.getTaxonNodeFilter().getSubtreeFilter().get(0).getUuid());
 108             }
 109
 110             @SuppressWarnings({ "unchecked", "rawtypes" })
 111             TaxonNodeOutStreamPartitioner<XmlExportState> partitioner = TaxonNodeOutStreamPartitioner.NewInstance(this,
 112                     state, state.getConfig().getTaxonNodeFilter(), 100, monitor, null);
 113
 114 //          handleMetaData(state);  //FIXME metadata;
 115             monitor.subTask("Start partitioning");
 116
 117             TaxonNode node = partitioner.next();
 118             while (node != null) {
 119                 handleTaxonNode(state, node);
 120                 node = partitioner.next();
 121             }
 122
 123             state.getProcessor().createFinalResult(state);
 124         } catch (Exception e) {
 125             state.getResult().addException(e,
 126                     "An unexpected error occurred in main method doInvoke() " + e.getMessage());
 127             e.printStackTrace();
 128         }
 129     }
 130
 131     private void handleTaxonNode(WfoContentExportState state, TaxonNode taxonNode) {
 132
 133         if (taxonNode == null) {
 134             // TODO 5 taxon node not found
 135             String message = "TaxonNode for given taxon node UUID not found.";
 136             state.getResult().addError(message);
 137         } else {
 138             try {
 139                 boolean exclude = filterTaxon(state, taxonNode);
 140
 141                 //handle taxon
 142                 String parentWfoId = getParentWfoId(state, taxonNode);
 143                 if (taxonNode.hasTaxon()) {
 144                     if (exclude) {
 145                         state.putTaxonNodeWfoId(taxonNode, parentWfoId); //always use parent instead
 146                     } else {
 147                         String wfoId = handleTaxon(state, taxonNode, parentWfoId);
 148                         state.putTaxonNodeWfoId(taxonNode, wfoId);
 149                     }
 150                     state.putTaxonNodeWfoId(taxonNode, parentWfoId);
 151                 }
 152             } catch (Exception e) {
 153                 state.getResult().addException(e, "An unexpected error occurred when handling taxonNode "
 154                         + taxonNode.getUuid() + ": " + e.getMessage() + e.getStackTrace());
 155             }
 156         }
 157     }
 158
 159     private String getParentWfoId(WfoContentExportState state, TaxonNode taxonNode) {
 160         TaxonNode parentNode = taxonNode.getParent();
 161         if (parentNode == null) {
 162             return null;
 163         }
 164         String wfoId = state.getTaxonNodeWfoId(parentNode);
 165         if (wfoId != null) {
 166             return wfoId;
 167         }else {
 168             wfoId = parentNode.getTaxon() == null ? null
 169                     : getWfoId(state, parentNode.getTaxon().getName(), true);
 170             if (wfoId != null) {
 171                 state.putTaxonNodeWfoId(parentNode, wfoId);
 172                 state.putTaxonWfoId(parentNode.getTaxon(), wfoId);
 173                 state.putNameWfoId(parentNode.getTaxon().getName(), wfoId);
 174             }
 175
 176             return wfoId;
 177         }
 178     }
 179
 180     private Set<Rank> allowedRanks = new HashSet<>();
 181     private boolean filterTaxon(WfoContentExportState state, TaxonNode taxonNode) {
 182         Taxon taxon = taxonNode.getTaxon();
 183         if (taxon == null) {
 184             return true;
 185         }
 186         TaxonName taxonName = taxon.getName();
 187         if (taxonName == null) {
 188             return true;
 189         }else if (taxonName.isHybridFormula()) {
 190             return true;
 191         }else {
 192             String wfoId = getWfoId(state, taxonName, false);
 193             if (wfoId == null) {
 194                 return true;
 195             }
 196         }
 197
 198         Rank rank = taxonName.getRank();
 199         if (rank == null) {
 200             return true;
 201         }else {
 202             if (allowedRanks.isEmpty()) {
 203                 allowedRanks.add(Rank.FAMILY());
 204                 allowedRanks.add(Rank.SUBFAMILY());
 205                 allowedRanks.add(Rank.TRIBE());
 206                 allowedRanks.add(Rank.SUBTRIBE());
 207                 allowedRanks.add(Rank.GENUS());
 208                 allowedRanks.add(Rank.SUBGENUS());
 209                 allowedRanks.add(Rank.SECTION_BOTANY());
 210                 allowedRanks.add(Rank.SUBSECTION_BOTANY());
 211                 allowedRanks.add(Rank.SPECIES());
 212                 allowedRanks.add(Rank.SUBSPECIES());
 213                 allowedRanks.add(Rank.VARIETY());
 214                 allowedRanks.add(Rank.SUBVARIETY());
 215                 allowedRanks.add(Rank.FORM());
 216                 allowedRanks.add(Rank.SUBFORM());
 217                 allowedRanks.add(Rank.INFRASPECIFICTAXON());
 218             }
 219             if (!allowedRanks.contains(rank)) {
 220                 //TODO 3 warn if this happens as such names should not have a wfo-id neither
 221                 return true;
 222             }
 223         }
 224         return false;
 225     }
 226
 227     /**
 228      * @return the WFO-ID of the taxon
 229      */
 230     private String handleTaxon(WfoContentExportState state, TaxonNode taxonNode, String parentWfoId) {
 231
 232         //check null
 233         if (taxonNode == null) {
 234             state.getResult().addError("The taxonNode was null.", "handleTaxon");
 235             state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 236             return null;
 237         }
 238         //check no taxon
 239         if (taxonNode.getTaxon() == null) {
 240             state.getResult().addError("There was a taxon node without a taxon: " + taxonNode.getUuid(),
 241                     "handleTaxon");
 242             state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 243             return null;
 244         }
 245
 246         //deproxy, just in case
 247         Taxon taxon = CdmBase.deproxy(taxonNode.getTaxon());
 248         String wfoId = null;
 249
 250         //handle taxon
 251         try {
 252
 253             //classification csvLine
 254             WfoContentExportTable table = WfoContentExportTable.CLASSIFICATION;
 255             String[] csvLine = new String[table.getSize()];
 256
 257             //accepted name
 258             TaxonName name = taxon.getName();
 259             wfoId = handleName(state, table, csvLine, name);
 260
 261             //... parentNameUsageID
 262             csvLine[table.getIndex(WfoContentExportTable.TAX_PARENT_ID)] = parentWfoId;
 263
 264             handleDescriptions(state, taxon);
 265
 266             //process taxon line
 267             state.getProcessor().put(table, taxon, csvLine);
 268
 269         } catch (Exception e) {
 270             e.printStackTrace();
 271             state.getResult().addException(e,
 272                     "An unexpected problem occurred when trying to export taxon with id " + taxon.getId() + " " + taxon.getTitleCache());
 273             state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 274             return null;
 275         }
 276
 277         return wfoId;
 278     }
 279
 280     private void handleDescriptions(WfoContentExportState state, Taxon taxon) {
 281
 282         //filtered descriptions
 283         Set<TaxonDescription> descriptions = taxon.getDescriptions().stream().filter(d->d.isPublish()).collect(Collectors.toSet());
 284         Stream<DescriptionElementBase> debStream = descriptions.stream().flatMap(d->d.getElements().stream());
 285
 286         SetMap<Feature,DescriptionElementBase> feature2DescriptionsMap = new SetMap<>();
 287         debStream.forEach(deb->feature2DescriptionsMap.putItem(deb.getFeature(), deb));
 288
 289         feature2DescriptionsMap.entrySet().stream().forEach(e->{
 290             Feature feature = e.getKey();
 291             e.getValue().forEach(deb->{
 292                 deb = CdmBase.deproxy(deb);
 293                 if (Feature.uuidDistribution.equals(feature.getUuid()) && deb.getClass().equals(Distribution.class)) {
 294                     handleDistribution(state, (Distribution)e.getValue(), taxon);
 295                 }else if (Feature.uuidCommonName.equals(feature.getUuid()) && deb.getClass().equals(CommonTaxonName.class)){
 296                     handleCommonName(state, (CommonTaxonName)e.getValue(), taxon);
 297                 }else if (Feature.uuidImage.equals(feature.getUuid())) {
 298                     //TODO 2 handle media
 299                 }else if (Feature.uuidHabitat.equals(feature.getUuid())) {
 300                     handleMeasurementOrFact(state, "http://kew.org/wcs/terms/habitat", deb, taxon);
 301                 }else if (Feature.uuidLifeform.equals(feature.getUuid())) {
 302                     handleMeasurementOrFact(state, "http://kew.org/wcs/terms/lifeform", deb, taxon);
 303                 }else if (Feature.uuidIucnStatus.equals(feature.getUuid())) {
 304                     handleMeasurementOrFact(state, "http://kew.org/wcs/terms/hreatStatus", deb, taxon);
 305                 }else {
 306                     //general description
 307                     handleDescription(state, deb, taxon);
 308                 }
 309             });
 310         });
 311     }
 312
 313     private void handleDescription(WfoContentExportState state, DescriptionElementBase deb, Taxon taxon) {
 314
 315         try {
 316             WfoContentExportTable table = WfoContentExportTable.DESCRIPTION;
 317             //TODO i18n
 318             List<Language> languages = new ArrayList<>();
 319             languages.add(Language.ENGLISH());
 320             languages.add(Language.FRENCH());
 321             languages.add(Language.SPANISH_CASTILIAN());
 322             languages.add(Language.GERMAN());
 323
 324             String[] csvLine = new String[table.getSize()];
 325
 326             //TODO 3 description types still need fine-tuning
 327             String type = getDescriptionTpe(state, deb);
 328             if (type == null) {
 329                 return;
 330             }
 331
 332             //description
 333             String text = null;
 334             if (deb instanceof TextData) {
 335                 TextData td = (TextData)deb;
 336
 337                 //TODO i18n
 338                 LanguageString ls = td.getPreferredLanguageString(languages, INCLUDE_UNPUBLISHED);
 339                 if (ls != null) {
 340                     text = ls.getText();
 341                     //language TODO
 342                 }
 343             } else if (deb instanceof CategoricalData) {
 344 //            DefaultCategoricalDescriptionBuilder builder = new DefaultCategoricalDescriptionBuilder();
 345 //            text = builder.build((CategoricalData)deb, languages);
 346                 //TODO which formatter to use
 347                 CategoricalDataFormatter formatter = CategoricalDataFormatter.NewInstance(null);
 348                 text = formatter.format(deb);
 349             } else {
 350                 //TODO other types or only message?
 351             }
 352             csvLine[table.getIndex(WfoContentExportTable.DESC_DESCRIPTION)] = text;
 353
 354             //audience TODO
 355             csvLine[table.getIndex(WfoContentExportTable.AUDIENCE)] = null;
 356
 357             //rights holder
 358             handleRightsHolder(state, deb, csvLine, table, taxon);
 359
 360             //created TODO
 361             handleCreated(state, deb, csvLine, table, taxon);
 362
 363             //creator
 364             handleCreator(state, deb, csvLine, table, taxon);
 365
 366             //source
 367             handleSource(state, deb, table);
 368
 369             //rights
 370             handleRights(state, null, csvLine, table, taxon);
 371
 372             //license
 373             handleLicense(state, null, csvLine, table, taxon);
 374         } catch (Exception e) {
 375             state.getResult().addException(e, "An unexpected error occurred when handling single description "
 376                     + cdmBaseStr(deb) + ": " + e.getMessage());
 377         }
 378     }
 379
 380     private String getDescriptionTpe(WfoContentExportState state, DescriptionElementBase deb) {
 381         try {
 382             return state.getTransformer().getCacheByFeature(deb.getFeature());
 383         } catch (UndefinedTransformerMethodException e) {
 384 //            e.printStackTrace();
 385             return null; //should not happen
 386         }
 387     }
 388
 389     private void handleCreator(WfoContentExportState state, DescriptionElementBase deb, String[] csvLine,
 390             WfoContentExportTable table, Taxon taxon) {
 391         // TODO Auto-generated method stub
 392     }
 393
 394     private void handleCreated(WfoContentExportState state, DescriptionElementBase deb, String[] csvLine,
 395             WfoContentExportTable table, Taxon taxon) {
 396         // TODO created
 397     }
 398
 399     private void handleRightsHolder(WfoContentExportState state, DescriptionElementBase deb,
 400             String[] csvLine, WfoContentExportTable table, Taxon taxon) {
 401         // TODO rightsholder
 402         csvLine[table.getIndex(WfoContentExportTable.RIGHTS_HOLDER)] = null;
 403
 404     }
 405
 406     private void handleMeasurementOrFact(WfoContentExportState state, String string, DescriptionElementBase deb,
 407             Taxon taxon) {
 408
 409     }
 410
 411     private void handleCommonName(WfoContentExportState state, CommonTaxonName commonName, Taxon taxon) {
 412         WfoContentExportTable table = WfoContentExportTable.VERNACULAR_NAME;
 413         //TODO i18n
 414         List<Language> languages = null;
 415         try {
 416             if (commonName instanceof CommonTaxonName) {
 417                 String[] csvLine = new String[table.getSize()];
 418 //                Distribution distribution = (Distribution) element;
 419 //                distributions.add(distribution);
 420
 421                 csvLine[table.getIndex(WfoContentExportTable.CN_VERNACULAR_NAME)] = commonName.getName();
 422
 423                 //language
 424                 if (commonName.getLanguage() != null) {
 425                     csvLine[table.getIndex(WfoContentExportTable.CN_VERNACULAR_NAME)] = commonName.getLanguage().getPreferredLabel(languages);
 426                 }
 427
 428                 //countryCode
 429                 NamedArea area = commonName.getArea();
 430                 if (area != null && area.getVocabulary() != null && area.getVocabulary().getUuid().equals(NamedArea.uuidCountryVocabulary)) {
 431                     String countryCode = ((Country)area).getIso3166_A2();
 432                     csvLine[table.getIndex(WfoContentExportTable.CN_COUNTRY_CODE)] = countryCode;
 433                 }
 434
 435                 //rights
 436                 handleRights(state, commonName, csvLine, table, taxon);
 437
 438                 //license
 439                 handleLicense(state, commonName, csvLine, table, taxon);
 440
 441                 //source
 442                 handleSource(state, commonName, table);
 443
 444                 state.getProcessor().put(table, commonName, csvLine);
 445             } else {
 446                 //TODO 1 is this handled elsewhere?
 447                 state.getResult()
 448                         .addError("The common name for the taxon " + taxon.getUuid()
 449                                 + " is not of type CommonTaxonName. Could not be exported. UUID of the common name: "
 450                                 + commonName.getUuid());
 451             }
 452         } catch (Exception e) {
 453             state.getResult().addException(e, "An unexpected error occurred when handling single common name "
 454                     + cdmBaseStr(commonName) + ": " + e.getMessage());
 455         }
 456     }
 457
 458     private void handleLicense(WfoContentExportState state, CommonTaxonName commonName,
 459             String[] csvLine, WfoContentExportTable table, Taxon taxon) {
 460         // TODO 1 handle License
 461
 462     }
 463
 464     private void handleRights(WfoContentExportState state, CommonTaxonName commonName,
 465             String[] csvLine, WfoContentExportTable table, Taxon taxon) {
 466         // TODO 1 handle rights
 467     }
 468
 469     private void handleDistribution(WfoContentExportState state, Distribution distribution, Taxon taxon) {
 470         WfoContentExportTable table = WfoContentExportTable.DISTRIBUTION;
 471         //TODO i18n
 472         List<Language> languages = null;
 473             try {
 474                 if (distribution instanceof Distribution) {
 475                     String[] csvLine = new String[table.getSize()];
 476 //                    Distribution distribution = (Distribution) element;
 477 //                    distributions.add(distribution);
 478                     NamedArea area = distribution.getArea();
 479
 480                     csvLine[table.getIndex(WfoContentExportTable.DIST_LOCALITY)] = area.getPreferredLabel(languages);
 481
 482                     //TDWG area
 483                     if (area.getVocabulary() !=null && area.getVocabulary().getUuid().equals(NamedArea.uuidTdwgAreaVocabulary)) {
 484                         String tdwgCode = area.getIdInVocabulary();
 485                         csvLine[table.getIndex(WfoContentExportTable.DIST_LOCATION_ID)] = tdwgCode;
 486                     }
 487
 488                     //countryCode
 489                     if (area.getVocabulary() !=null && area.getVocabulary().getUuid().equals(NamedArea.uuidCountryVocabulary)) {
 490                         String countryCode = ((Country)area).getIso3166_A2();
 491                         csvLine[table.getIndex(WfoContentExportTable.DIST_COUNTRY_CODE)] = countryCode;
 492                     }
 493
 494                     if (distribution.getStatus() != null) {
 495                         PresenceAbsenceTerm status = distribution.getStatus();
 496                         csvLine[table.getIndex(WfoContentExportTable.DIST_ESTABLISHMENT_MEANS)] = status.getPreferredLabel(languages);
 497                     }
 498                     //source
 499                     handleSource(state, distribution, table);
 500
 501                     //occurrencRemarks
 502                     //TODO 5 occurrence remarks correct?
 503                     csvLine[table.getIndex(WfoContentExportTable.DIST_OCCURRENCE_REMARKS)] = createAnnotationsString(distribution.getAnnotations());
 504
 505                     state.getProcessor().put(table, distribution, csvLine);
 506                 } else {
 507                     //TODO 1 is this handled elsewhere?
 508                     state.getResult()
 509                             .addError("The distribution description for the taxon " + taxon.getUuid()
 510                                     + " is not of type distribution. Could not be exported. UUID of the description element: "
 511                                     + distribution.getUuid());
 512                 }
 513             } catch (Exception e) {
 514                 state.getResult().addException(e, "An unexpected error occurred when handling single distribution "
 515                         + cdmBaseStr(distribution) + ": " + e.getMessage());
 516             }
 517     }
 518
 519     private boolean isUrl(String url) {
 520         try {
 521             if (url.startsWith("http")) {
 522                 URI.fromString(url);
 523                 return true;
 524             }
 525         } catch (Exception e) {
 526             //exception should return false
 527         }
 528         return false;
 529     }
 530
 531     private String toIsoDate(TimePeriod mediaCreated) {
 532         //TODO 2 date, what if end or freetext exist?
 533         Partial partial = mediaCreated.getStart();
 534         if (partial == null || !partial.isSupported(DateTimeFieldType.year())
 535                 || !partial.isSupported(DateTimeFieldType.monthOfYear()) && partial.isSupported(DateTimeFieldType.dayOfMonth())) {
 536             //TODO 2 date, log warning, also if mediaCreated.getEnd() != null or so
 537             return null;
 538         } else {
 539             DateTimeFormatter formatter = new DateTimeFormatterBuilder()
 540                     .appendYear(4, 4).appendLiteral('-')
 541                     .appendMonthOfYear(2).appendLiteral('-')
 542                     .appendDayOfMonth(2)
 543                     .toFormatter();
 544             return partial.toString(formatter);
 545         }
 546     }
 547
 548     /**
 549      * transforms the given date to an iso date
 550      */
 551     protected String toIsoDate(DateTime dateTime) {
 552         if (dateTime == null) {
 553             return null;
 554         }
 555         DateTimeFormatter formatter = new DateTimeFormatterBuilder()
 556                 .appendYear(4, 4).appendLiteral('-')
 557                 .appendMonthOfYear(2).appendLiteral('-')
 558                 .appendDayOfMonth(2)
 559                 .toFormatter();
 560         return formatter.print(dateTime);
 561     }
 562
 563     private String getRemarks(AnnotatableEntity entity) {
 564         String remarks = null;
 565         for (Annotation a : entity.getAnnotations()) {
 566             //TODO 3 handle other annotation types
 567             if (AnnotationType.EDITORIAL().equals(a.getAnnotationType())
 568                     && CdmUtils.isNotBlank(a.getText())){
 569                 remarks = CdmUtils.concat(";", remarks, a.getText());
 570             }
 571         }
 572         return remarks;
 573     }
 574
 575 //    private void handleMetaData(ColDpExportState state) {
 576 //        ColDpExportTable table = ColDpExportTable.METADATA;
 577 //        String[] csvLine = new String[table.getSize()];
 578 ////        csvLine[table.getIndex(CdmLightExportTable.INSTANCE_ID)] = state.getConfig().getInctanceId();
 579 ////        csvLine[table.getIndex(CdmLightExportTable.INSTANCE_NAME)] = state.getConfig().getInstanceName();
 580 //        csvLine[table.getIndex(ColDpExportTable.DATASET_BASE_URL)] = state.getConfig().getBase_url();
 581 //        csvLine[table.getIndex(ColDpExportTable.DATASET_CONTRIBUTOR)] = state.getConfig().getContributor();
 582 //        csvLine[table.getIndex(ColDpExportTable.DATASET_CREATOR)] = state.getConfig().getCreator();
 583 //        csvLine[table.getIndex(ColDpExportTable.DATASET_DESCRIPTION)] = state.getConfig().getDescription();
 584 //        csvLine[table.getIndex(ColDpExportTable.DATASET_DOWNLOAD_LINK)] = state.getConfig().getDataset_download_link();
 585 //        csvLine[table.getIndex(ColDpExportTable.DATASET_KEYWORDS)] = state.getConfig().getKeywords();
 586 //        csvLine[table.getIndex(ColDpExportTable.DATASET_LANDINGPAGE)] = state.getConfig().getDataSet_landing_page();
 587 //
 588 //        csvLine[table.getIndex(ColDpExportTable.DATASET_LANGUAGE)] = state.getConfig().getLanguage() != null? state.getConfig().getLanguage().getLabel(): null;
 589 //        csvLine[table.getIndex(ColDpExportTable.DATASET_LICENCE)] = state.getConfig().getLicence();
 590 //        csvLine[table.getIndex(ColDpExportTable.DATASET_LOCATION)] = state.getConfig().getLocation();
 591 //        csvLine[table.getIndex(ColDpExportTable.DATASET_RECOMMENDED_CITATTION)] = state.getConfig().getRecommended_citation();
 592 //        csvLine[table.getIndex(ColDpExportTable.DATASET_TITLE)] = state.getConfig().getTitle();
 593 //        state.getProcessor().put(table, "", csvLine);
 594 //    }
 595
 596     private String createMultilanguageString(Map<Language, LanguageString> multilanguageText) {
 597         String text = "";
 598         int index = multilanguageText.size();
 599         for (LanguageString langString : multilanguageText.values()) {
 600             text += langString.getText();
 601             if (index > 1) {
 602                 text += "; ";
 603             }
 604             index--;
 605         }
 606         return text;
 607     }
 608
 609     private String createAnnotationsString(Set<Annotation> annotations) {
 610         StringBuffer strBuff = new StringBuffer();
 611
 612         for (Annotation ann : annotations) {
 613             if (ann.getAnnotationType() == null || !ann.getAnnotationType().equals(AnnotationType.TECHNICAL())) {
 614                 strBuff.append(ann.getText());
 615                 strBuff.append("; ");
 616             }
 617         }
 618
 619         if (strBuff.length() > 2) {
 620             return strBuff.substring(0, strBuff.length() - 2);
 621         } else {
 622             return null;
 623         }
 624     }
 625
 626     private void handleSource(WfoContentExportState state, DescriptionElementBase element,
 627             WfoContentExportTable factsTable) {
 628 //        ColDpExportTable table = ColDpExportTable.FACT_SOURCES;
 629 //        try {
 630 //            Set<DescriptionElementSource> sources = element.getSources();
 631 //
 632 //            for (DescriptionElementSource source : sources) {
 633 //                if (!(source.getType().equals(OriginalSourceType.Import)
 634 //                        && state.getConfig().isExcludeImportSources())) {
 635 //                    String[] csvLine = new String[table.getSize()];
 636 //                    Reference ref = source.getCitation();
 637 //                    if ((ref == null) && (source.getNameUsedInSource() == null)) {
 638 //                        continue;
 639 //                    }
 640 //                    if (ref != null) {
 641 //                        if (!state.getReferenceStore().contains(ref.getUuid())) {
 642 //                            handleReference(state, ref);
 643 //
 644 //                        }
 645 //                        csvLine[table.getIndex(ColDpExportTable.REFERENCE_FK)] = getId(state, ref);
 646 //                    }
 647 //                    csvLine[table.getIndex(ColDpExportTable.FACT_FK)] = getId(state, element);
 648 //
 649 //                    csvLine[table.getIndex(ColDpExportTable.NAME_IN_SOURCE_FK)] = getId(state,
 650 //                            source.getNameUsedInSource());
 651 //                    csvLine[table.getIndex(ColDpExportTable.FACT_TYPE)] = factsTable.getTableName();
 652 //                    if (StringUtils.isBlank(csvLine[table.getIndex(ColDpExportTable.REFERENCE_FK)])
 653 //                            && StringUtils.isBlank(csvLine[table.getIndex(ColDpExportTable.NAME_IN_SOURCE_FK)])) {
 654 //                        continue;
 655 //                    }
 656 //                    state.getProcessor().put(table, source, csvLine);
 657 //                }
 658 //            }
 659 //        } catch (Exception e) {
 660 //            state.getResult().addException(e, "An unexpected error occurred when handling single source "
 661 //                    + cdmBaseStr(element) + ": " + e.getMessage());
 662 //        }
 663     }
 664
 665     private String getTitleCache(IIdentifiableEntity identEntity) {
 666         if (identEntity == null) {
 667             return "";
 668         }
 669         // TODO 3 titleCache refresh?
 670         return identEntity.getTitleCache();
 671     }
 672
 673     private String getId(WfoContentExportState state, ICdmBase cdmBase) {
 674         if (cdmBase == null) {
 675             return "";
 676         }
 677         // TODO 4 id type, make configurable
 678         return cdmBase.getUuid().toString();
 679     }
 680
 681     private String handleName(WfoContentExportState state, WfoContentExportTable table, String[] csvLine,
 682             TaxonName name) {
 683
 684         name = CdmBase.deproxy(name);
 685         if (name == null || state.getNameStore().containsKey(name.getId())) {
 686             if (name == null) {
 687                 state.getResult().addError("No name was given for taxon.", "handleName");
 688                 state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 689             }
 690             return null;
 691         }
 692
 693         String wfoId = null;
 694         try {
 695
 696             state.getNameStore().put(name.getId(), name.getUuid());
 697
 698             //taxonID
 699             wfoId = getWfoId(state, name, true);
 700             if (isBlank(wfoId)) {
 701                 String message = "No WFO-ID given for taxon " + name.getTitleCache() + ". Taxon ignored";
 702                 state.getResult().addError(message);
 703                 state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 704                 return null;
 705             }else {
 706                 csvLine[table.getIndex(WfoContentExportTable.TAXON_ID)] = wfoId;
 707             }
 708
 709             //scientificNameID
 710             //TODO 9 add IPNI ID if exists
 711             boolean warnIfNotExists = false;
 712             csvLine[table.getIndex(WfoContentExportTable.NAME_SCIENTIFIC_NAME_ID)] = getIpniId(state, name, warnIfNotExists);
 713
 714             //scientificName
 715             if (name.isProtectedTitleCache()) {
 716                 //TODO 7 make it configurable if we should always take titleCache if titleCache is protected, as nameCache may not necessarily
 717                 //     have complete data if titleCache is protected as it is considered to be irrelevant or at least preliminary
 718                 String message = "";
 719                 if (StringUtils.isNotEmpty(name.getNameCache())) {
 720                     csvLine[table.getIndex(WfoContentExportTable.NAME_SCIENTIFIC_NAME)] = name.getNameCache();
 721                     message = "ScientificName: Name cache " + name.getNameCache() + " used for name with protected titleCache " +  name.getTitleCache();
 722                 }else {
 723                     csvLine[table.getIndex(WfoContentExportTable.NAME_SCIENTIFIC_NAME)] = name.getTitleCache();
 724                     message = "ScientificName: Name has protected titleCache and no explicit nameCache: " +  name.getTitleCache();
 725                 }
 726                 state.getResult().addWarning(message);  //TODO 7 add location to warning
 727             } else {
 728                 csvLine[table.getIndex(WfoContentExportTable.NAME_SCIENTIFIC_NAME)] = name.getNameCache();
 729             }
 730
 731             //rank
 732             Rank rank = name.getRank();
 733             String rankStr = state.getTransformer().getCacheByRank(rank);
 734             if (rankStr == null) {
 735                 String message = rank == null ? "No rank" : ("Rank not supported by WFO:" + rank.getLabel())
 736                         + "Taxon not handled in export: " + name.getTitleCache();
 737                 state.getResult().addWarning(message);  //TODO 2 warning sufficient for missing rank? + location
 738                 return wfoId;
 739             }
 740             csvLine[table.getIndex(WfoContentExportTable.RANK)] = rankStr;
 741
 742             //scientificNameAuthorship
 743             //TODO 3 handle empty authorship cache warning
 744             csvLine[table.getIndex(WfoContentExportTable.NAME_AUTHORSHIP)] = name.getAuthorshipCache();
 745
 746             //family
 747             //TODO 2 family handling
 748             csvLine[table.getIndex(WfoContentExportTable.TAX_FAMILY)] = state.getFamilyStr();
 749
 750             //name parts
 751             csvLine[table.getIndex(WfoContentExportTable.TAX_GENUS)] = name.isSupraGeneric()? null : name.getGenusOrUninomial();
 752             csvLine[table.getIndex(WfoContentExportTable.NAME_SPECIFIC_EPITHET)] = name.getSpecificEpithet();
 753             csvLine[table.getIndex(WfoContentExportTable.NAME_INFRASPECIFIC_EPITHET)] = name.getInfraSpecificEpithet();
 754
 755             //nom. ref
 756             String nomRef = NomenclaturalSourceFormatter.INSTANCE().format(name.getNomenclaturalSource());
 757             csvLine[table.getIndex(WfoContentExportTable.NAME_PUBLISHED_IN)] = nomRef;
 758
 759         } catch (Exception e) {
 760             state.getResult().addException(e,
 761                     "An unexpected error occurred when handling the name " + cdmBaseStr(name) + ": " + name.getTitleCache() + ": " + e.getMessage());
 762
 763             e.printStackTrace();
 764         }
 765
 766         return wfoId;
 767     }
 768
 769     private String getIpniId(WfoContentExportState state, TaxonName name, boolean warnIfNotExists) {
 770         Identifier ipniId = name.getIdentifier(IdentifierType.uuidIpniNameIdentifier);
 771         if (ipniId == null && warnIfNotExists) {
 772             String message = "No ipni-id given for name: " + name.getTitleCache()+"/"+ name.getUuid();
 773             state.getResult().addWarning(message);  //TODO 5 data location
 774         }
 775         return ipniId == null ? null : ipniId.getIdentifier();
 776     }
 777
 778     private String getWfoId(WfoContentExportState state, TaxonName name, boolean warnIfNotExists) {
 779         Identifier wfoId = name.getIdentifier(IdentifierType.uuidWfoNameIdentifier);
 780         if (wfoId == null && warnIfNotExists) {
 781             String message = "No wfo-id given for name: " + name.getTitleCache()+"/"+ name.getUuid();
 782             state.getResult().addWarning(message);  //TODO 5 data location
 783         }
 784         return wfoId == null ? null : wfoId.getIdentifier();
 785     }
 786
 787     private String makeNameStatus(WfoContentExportState state, TaxonName name) {
 788         try {
 789
 790             //TODO 1 what is with dubium
 791             if (name.isLegitimate()) {
 792                 if (name.isConserved()) {
 793                     return "Conserved";
 794                 }else {
 795                     return "Valid";
 796                 }
 797             } else if (name.isRejected()) {
 798                 return "Rejected";
 799             } else if (name.isIllegitimate()) {
 800                 return "Illegitimate";
 801             } else if (name.isInvalid()) {
 802                 //TODO 2 handle original spellings for name status
 803                 if (name.isOrthographicVariant()) {
 804                     return "orthografia";
 805                 }else {
 806                     return "invalid";
 807                 }
 808             } else {
 809                 String message = "Unhandled name status case for name: " + name.getTitleCache() +
 810                         ". Status not handled correctly.";
 811                 state.getResult().addWarning(message);
 812                 return "undefined";
 813             }
 814         } catch (Exception e) {
 815             state.getResult().addException(e, "An unexpected error occurred when extracting status string for "
 816                     + cdmBaseStr(name) + ": " + e.getMessage());
 817             return "";
 818         }
 819     }
 820
 821     private void handleHomotypicalGroup(WfoContentExportState state, HomotypicalGroup group, Taxon acceptedTaxon) {
 822         try {
 823
 824             List<TaxonName> typifiedNames = new ArrayList<>();
 825             if (acceptedTaxon != null){
 826                 List<Synonym> synonymsInGroup = acceptedTaxon.getSynonymsInGroup(group);
 827                 if (group.equals(acceptedTaxon.getHomotypicGroup())){
 828                     typifiedNames.add(acceptedTaxon.getName());
 829                 }
 830                 synonymsInGroup.stream().forEach(synonym -> typifiedNames.add(CdmBase.deproxy(synonym.getName())));
 831             }
 832
 833
 834             TaxonName firstname = null;
 835             for (TaxonName name: typifiedNames){
 836                 Iterator<Taxon> taxa = name.getTaxa().iterator();
 837                 while(taxa.hasNext()){
 838                     Taxon taxon = taxa.next();
 839                     if(!(taxon.isMisapplication() || taxon.isProparteSynonym())){
 840                         firstname = name;
 841                         break;
 842                     }
 843                 }
 844             }
 845
 846         } catch (Exception e) {
 847             state.getResult().addException(e, "An unexpected error occurred when handling homotypic group "
 848                     + cdmBaseStr(group) + ": " + e.getMessage());
 849         }
 850     }
 851
 852     private void handleReference(WfoContentExportState state, Reference reference) {
 853         try {
 854             if (state.getReferenceStore().contains(reference.getUuid())) {
 855                 return;
 856             }
 857             reference = CdmBase.deproxy(reference);
 858
 859             state.addReferenceToStore(reference);
 860             WfoContentExportTable table = WfoContentExportTable.REFERENCE;
 861             String[] csvLine = new String[table.getSize()];
 862
 863             csvLine[table.getIndex(WfoContentExportTable.IDENTIFIER)] = getId(state, reference);
 864
 865             //TODO 2 correct?, ref biblio citation
 866             csvLine[table.getIndex(WfoContentExportTable.REF_BIBLIO_CITATION)] = reference.getCitation();
 867
 868             //TODO 1 uri (doi, uri or ext_link
 869 //            csvLine[table.getIndex(WfoContentExportTable.REF_DOI)] = reference.getDoiString();
 870 //
 871 //            //TODO 2 reference link link (=> external link)
 872 ////            csvLine[table.getIndex(ColDpExportTable.LINK)] = null;
 873 //            if (reference.getUri() != null) {
 874 //                csvLine[table.getIndex(WfoContentExportTable.LINK)] = reference.getUri().toString();
 875 //            }
 876
 877             state.getProcessor().put(table, reference, csvLine);
 878         } catch (Exception e) {
 879             e.printStackTrace();
 880             state.getResult().addException(e, "An unexpected error occurred when handling reference "
 881                     + cdmBaseStr(reference) + ": " + e.getMessage());
 882         }
 883     }
 884
 885     /**
 886      * Returns a string representation of the {@link CdmBase cdmBase} object for
 887      * result messages.
 888      */
 889     private String cdmBaseStr(CdmBase cdmBase) {
 890         if (cdmBase == null) {
 891             return "-no object available-";
 892         } else {
 893             return cdmBase.getClass().getSimpleName() + ": " + cdmBase.getUuid();
 894         }
 895     }
 896
 897     @Override
 898     protected boolean doCheck(WfoContentExportState state) {
 899         return false;
 900     }
 901
 902     @Override
 903     protected boolean isIgnore(WfoContentExportState state) {
 904         return false;
 905     }
 906 }