cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/wfo/out/WfoContentExport.java

   1 /**
   2 * Copyright (C) 2017 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.io.wfo.out;
  10
  11 import java.io.File;
  12 import java.util.ArrayList;
  13 import java.util.HashSet;
  14 import java.util.Iterator;
  15 import java.util.List;
  16 import java.util.Map;
  17 import java.util.Set;
  18 import java.util.stream.Collectors;
  19 import java.util.stream.Stream;
  20
  21 import org.apache.commons.lang3.StringUtils;
  22 import org.joda.time.DateTime;
  23 import org.joda.time.DateTimeFieldType;
  24 import org.joda.time.Partial;
  25 import org.joda.time.format.DateTimeFormatter;
  26 import org.joda.time.format.DateTimeFormatterBuilder;
  27 import org.springframework.stereotype.Component;
  28
  29 import eu.etaxonomy.cdm.common.CdmUtils;
  30 import eu.etaxonomy.cdm.common.SetMap;
  31 import eu.etaxonomy.cdm.common.URI;
  32 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  33 import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
  34 import eu.etaxonomy.cdm.format.description.CategoricalDataFormatter;
  35 import eu.etaxonomy.cdm.format.reference.NomenclaturalSourceFormatter;
  36 import eu.etaxonomy.cdm.io.common.CdmExportBase;
  37 import eu.etaxonomy.cdm.io.common.ExportResult.ExportResultState;
  38 import eu.etaxonomy.cdm.io.common.TaxonNodeOutStreamPartitioner;
  39 import eu.etaxonomy.cdm.io.common.XmlExportState;
  40 import eu.etaxonomy.cdm.io.common.mapping.out.IExportTransformer;
  41 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  42 import eu.etaxonomy.cdm.model.common.Annotation;
  43 import eu.etaxonomy.cdm.model.common.AnnotationType;
  44 import eu.etaxonomy.cdm.model.common.CdmBase;
  45 import eu.etaxonomy.cdm.model.common.ICdmBase;
  46 import eu.etaxonomy.cdm.model.common.IIdentifiableEntity;
  47 import eu.etaxonomy.cdm.model.common.Identifier;
  48 import eu.etaxonomy.cdm.model.common.Language;
  49 import eu.etaxonomy.cdm.model.common.LanguageString;
  50 import eu.etaxonomy.cdm.model.common.TimePeriod;
  51 import eu.etaxonomy.cdm.model.description.CategoricalData;
  52 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  53 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  54 import eu.etaxonomy.cdm.model.description.Distribution;
  55 import eu.etaxonomy.cdm.model.description.Feature;
  56 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  57 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  58 import eu.etaxonomy.cdm.model.description.TextData;
  59 import eu.etaxonomy.cdm.model.location.Country;
  60 import eu.etaxonomy.cdm.model.location.NamedArea;
  61 import eu.etaxonomy.cdm.model.name.HomotypicalGroup;
  62 import eu.etaxonomy.cdm.model.name.Rank;
  63 import eu.etaxonomy.cdm.model.name.TaxonName;
  64 import eu.etaxonomy.cdm.model.reference.Reference;
  65 import eu.etaxonomy.cdm.model.taxon.Classification;
  66 import eu.etaxonomy.cdm.model.taxon.Synonym;
  67 import eu.etaxonomy.cdm.model.taxon.Taxon;
  68 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  69 import eu.etaxonomy.cdm.model.term.IdentifierType;
  70
  71 /**
  72  * Classification or taxon tree exporter into WFO Content format.
  73  *
  74  * @author a.mueller
  75  * @since 2024-01-30
  76  */
  77 @Component
  78 public class WfoContentExport
  79         extends CdmExportBase<WfoContentExportConfigurator,WfoContentExportState,IExportTransformer,File>{
  80
  81     private static final long serialVersionUID = -4560488499411723333L;
  82
  83     public WfoContentExport() {
  84         this.ioName = this.getClass().getSimpleName();
  85     }
  86
  87     @Override
  88     public long countSteps(WfoContentExportState state) {
  89         TaxonNodeFilter filter = state.getConfig().getTaxonNodeFilter();
  90         return getTaxonNodeService().count(filter);
  91     }
  92
  93     @Override
  94     protected void doInvoke(WfoContentExportState state) {
  95
  96         try {
  97             IProgressMonitor monitor = state.getConfig().getProgressMonitor();
  98             WfoContentExportConfigurator config = state.getConfig();
  99
 100             //set root node
 101             if (config.getTaxonNodeFilter().hasClassificationFilter()) {
 102                 Classification classification = getClassificationService()
 103                         .load(config.getTaxonNodeFilter().getClassificationFilter().get(0).getUuid());
 104                 state.setRootId(classification.getRootNode().getUuid());
 105             } else if (config.getTaxonNodeFilter().hasSubtreeFilter()) {
 106                 state.setRootId(config.getTaxonNodeFilter().getSubtreeFilter().get(0).getUuid());
 107             }
 108
 109             @SuppressWarnings({ "unchecked", "rawtypes" })
 110             TaxonNodeOutStreamPartitioner<XmlExportState> partitioner = TaxonNodeOutStreamPartitioner.NewInstance(this,
 111                     state, state.getConfig().getTaxonNodeFilter(), 100, monitor, null);
 112
 113 //          handleMetaData(state);  //FIXME metadata;
 114             monitor.subTask("Start partitioning");
 115
 116             TaxonNode node = partitioner.next();
 117             while (node != null) {
 118                 handleTaxonNode(state, node);
 119                 node = partitioner.next();
 120             }
 121
 122             state.getProcessor().createFinalResult(state);
 123         } catch (Exception e) {
 124             state.getResult().addException(e,
 125                     "An unexpected error occurred in main method doInvoke() " + e.getMessage());
 126             e.printStackTrace();
 127         }
 128     }
 129
 130     private void handleTaxonNode(WfoContentExportState state, TaxonNode taxonNode) {
 131
 132         if (taxonNode == null) {
 133             // TODO 5 taxon node not found
 134             String message = "TaxonNode for given taxon node UUID not found.";
 135             state.getResult().addError(message);
 136         } else {
 137             try {
 138                 boolean exclude = filterTaxon(state, taxonNode);
 139
 140                 //handle taxon
 141                 String parentWfoId = getParentWfoId(state, taxonNode);
 142                 if (taxonNode.hasTaxon()) {
 143                     if (exclude) {
 144                         state.putTaxonNodeWfoId(taxonNode, parentWfoId); //always use parent instead
 145                     } else {
 146                         String wfoId = handleTaxon(state, taxonNode, parentWfoId);
 147                         state.putTaxonNodeWfoId(taxonNode, wfoId);
 148                     }
 149                     state.putTaxonNodeWfoId(taxonNode, parentWfoId);
 150                 }
 151             } catch (Exception e) {
 152                 state.getResult().addException(e, "An unexpected error occurred when handling taxonNode "
 153                         + taxonNode.getUuid() + ": " + e.getMessage() + e.getStackTrace());
 154             }
 155         }
 156     }
 157
 158     private String getParentWfoId(WfoContentExportState state, TaxonNode taxonNode) {
 159         TaxonNode parentNode = taxonNode.getParent();
 160         if (parentNode == null) {
 161             return null;
 162         }
 163         String wfoId = state.getTaxonNodeWfoId(parentNode);
 164         if (wfoId != null) {
 165             return wfoId;
 166         }else {
 167             wfoId = parentNode.getTaxon() == null ? null
 168                     : getWfoId(state, parentNode.getTaxon().getName(), true);
 169             if (wfoId != null) {
 170                 state.putTaxonNodeWfoId(parentNode, wfoId);
 171                 state.putTaxonWfoId(parentNode.getTaxon(), wfoId);
 172                 state.putNameWfoId(parentNode.getTaxon().getName(), wfoId);
 173             }
 174
 175             return wfoId;
 176         }
 177     }
 178
 179     private Set<Rank> allowedRanks = new HashSet<>();
 180     private boolean filterTaxon(WfoContentExportState state, TaxonNode taxonNode) {
 181         Taxon taxon = taxonNode.getTaxon();
 182         if (taxon == null) {
 183             return true;
 184         }
 185         TaxonName taxonName = taxon.getName();
 186         if (taxonName == null) {
 187             return true;
 188         }else if (taxonName.isHybridFormula()) {
 189             return true;
 190         }else {
 191             String wfoId = getWfoId(state, taxonName, false);
 192             if (wfoId == null) {
 193                 return true;
 194             }
 195         }
 196
 197         Rank rank = taxonName.getRank();
 198         if (rank == null) {
 199             return true;
 200         }else {
 201             if (allowedRanks.isEmpty()) {
 202                 allowedRanks.add(Rank.FAMILY());
 203                 allowedRanks.add(Rank.SUBFAMILY());
 204                 allowedRanks.add(Rank.TRIBE());
 205                 allowedRanks.add(Rank.SUBTRIBE());
 206                 allowedRanks.add(Rank.GENUS());
 207                 allowedRanks.add(Rank.SUBGENUS());
 208                 allowedRanks.add(Rank.SECTION_BOTANY());
 209                 allowedRanks.add(Rank.SPECIES());
 210                 allowedRanks.add(Rank.SUBSPECIES());
 211                 allowedRanks.add(Rank.VARIETY());
 212                 allowedRanks.add(Rank.SUBVARIETY());
 213                 allowedRanks.add(Rank.FORM());
 214                 allowedRanks.add(Rank.SUBFORM());
 215                 allowedRanks.add(Rank.INFRASPECIFICTAXON());
 216             }
 217             if (!allowedRanks.contains(rank)) {
 218                 //TODO 3 warn if this happens as such names should not have a wfo-id neither
 219                 return true;
 220             }
 221         }
 222         return false;
 223     }
 224
 225     /**
 226      * @return the WFO-ID of the taxon
 227      */
 228     private String handleTaxon(WfoContentExportState state, TaxonNode taxonNode, String parentWfoId) {
 229
 230         //check null
 231         if (taxonNode == null) {
 232             state.getResult().addError("The taxonNode was null.", "handleTaxon");
 233             state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 234             return null;
 235         }
 236         //check no taxon
 237         if (taxonNode.getTaxon() == null) {
 238             state.getResult().addError("There was a taxon node without a taxon: " + taxonNode.getUuid(),
 239                     "handleTaxon");
 240             state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 241             return null;
 242         }
 243
 244         //deproxy, just in case
 245         Taxon taxon = CdmBase.deproxy(taxonNode.getTaxon());
 246         String wfoId = null;
 247
 248         //handle taxon
 249         try {
 250
 251             //classification csvLine
 252             WfoContentExportTable table = WfoContentExportTable.CLASSIFICATION;
 253             String[] csvLine = new String[table.getSize()];
 254
 255             //accepted name
 256             TaxonName name = taxon.getName();
 257             wfoId = handleName(state, table, csvLine, name);
 258
 259             //... parentNameUsageID
 260             csvLine[table.getIndex(WfoContentExportTable.TAX_PARENT_ID)] = parentWfoId;
 261
 262             handleDescriptions(state, taxon);
 263
 264             //process taxon line
 265             state.getProcessor().put(table, taxon, csvLine);
 266
 267         } catch (Exception e) {
 268             e.printStackTrace();
 269             state.getResult().addException(e,
 270                     "An unexpected problem occurred when trying to export taxon with id " + taxon.getId() + " " + taxon.getTitleCache());
 271             state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 272             return null;
 273         }
 274
 275         return wfoId;
 276     }
 277
 278     private void handleDescriptions(WfoContentExportState state, Taxon taxon) {
 279
 280         //filtered descriptions
 281         Set<TaxonDescription> descriptions = taxon.getDescriptions().stream().filter(d->d.isPublish()).collect(Collectors.toSet());
 282         Stream<DescriptionElementBase> debStream = descriptions.stream().flatMap(d->d.getElements().stream());
 283
 284         SetMap<Feature,DescriptionElementBase> feature2DescriptionsMap = new SetMap<>();
 285         debStream.forEach(deb->feature2DescriptionsMap.putItem(deb.getFeature(), deb));
 286
 287         feature2DescriptionsMap.entrySet().stream().forEach(e->{
 288             Feature feature = e.getKey();
 289             e.getValue().forEach(deb->{
 290                 deb = CdmBase.deproxy(deb);
 291                 if (Feature.uuidDistribution.equals(feature.getUuid()) && deb.getClass().equals(Distribution.class)) {
 292                     handleDistribution(state, (Distribution)e.getValue(), taxon);
 293                 }else if (Feature.uuidCommonName.equals(feature.getUuid()) && deb.getClass().equals(CommonTaxonName.class)){
 294                     handleCommonName(state, (CommonTaxonName)e.getValue(), taxon);
 295                 }else if (Feature.uuidImage.equals(feature.getUuid())) {
 296                     //TODO 2 handle media
 297                 }else if (Feature.uuidHabitat.equals(feature.getUuid())) {
 298                     handleMeasurementOrFact(state, "http://kew.org/wcs/terms/habitat", deb, taxon);
 299                 }else if (Feature.uuidLifeform.equals(feature.getUuid())) {
 300                     handleMeasurementOrFact(state, "http://kew.org/wcs/terms/lifeform", deb, taxon);
 301                 }else if (Feature.uuidIucnStatus.equals(feature.getUuid())) {
 302                     handleMeasurementOrFact(state, "http://kew.org/wcs/terms/hreatStatus", deb, taxon);
 303                 }else {
 304                     //general description
 305                     handleDescription(state, deb, taxon);
 306                 }
 307             });
 308         });
 309     }
 310
 311     private void handleDescription(WfoContentExportState state, DescriptionElementBase deb, Taxon taxon) {
 312         WfoContentExportTable table = WfoContentExportTable.DESCRIPTION;
 313         //TODO i18n
 314         List<Language> languages = new ArrayList<>();
 315         languages.add(Language.ENGLISH());
 316         languages.add(Language.FRENCH());
 317         languages.add(Language.SPANISH_CASTILIAN());
 318         languages.add(Language.GERMAN());
 319
 320         String[] csvLine = new String[table.getSize()];
 321
 322         //type
 323         //TODO 1 description type
 324
 325         //description
 326         String text = null;
 327         if (deb instanceof TextData) {
 328             TextData td = (TextData)deb;
 329
 330             //TODO i18n
 331             LanguageString ls = td.getPreferredLanguageString(languages, INCLUDE_UNPUBLISHED);
 332             if (ls != null) {
 333                 text = ls.getText();
 334                 //language TODO
 335             }
 336         } else if (deb instanceof CategoricalData) {
 337 //            DefaultCategoricalDescriptionBuilder builder = new DefaultCategoricalDescriptionBuilder();
 338 //            text = builder.build((CategoricalData)deb, languages);
 339             //TODO which formatter to use
 340             CategoricalDataFormatter formatter = CategoricalDataFormatter.NewInstance(null);
 341             text = formatter.format(deb);
 342         } else {
 343             //TODO other types or only message?
 344         }
 345         csvLine[table.getIndex(WfoContentExportTable.DESC_DESCRIPTION)] = text;
 346
 347         //audience TODO
 348         csvLine[table.getIndex(WfoContentExportTable.AUDIENCE)] = null;
 349
 350         //rights holder
 351         handleRightsHolder(state, deb, csvLine, table, taxon);
 352
 353         //created TODO
 354         handleCreated(state, deb, csvLine, table, taxon);
 355
 356         //creator
 357         handleCreator(state, deb, csvLine, table, taxon);
 358
 359         //source
 360         handleSource(state, deb, table);
 361
 362         //rights
 363         handleRights(state, null, csvLine, table, taxon);
 364
 365         //license
 366         handleLicense(state, null, csvLine, table, taxon);
 367     }
 368
 369     private void handleCreator(WfoContentExportState state, DescriptionElementBase deb, String[] csvLine,
 370             WfoContentExportTable table, Taxon taxon) {
 371         // TODO Auto-generated method stub
 372     }
 373
 374     private void handleCreated(WfoContentExportState state, DescriptionElementBase deb, String[] csvLine,
 375             WfoContentExportTable table, Taxon taxon) {
 376         // TODO created
 377     }
 378
 379     private void handleRightsHolder(WfoContentExportState state, DescriptionElementBase deb,
 380             String[] csvLine, WfoContentExportTable table, Taxon taxon) {
 381         // TODO rightsholder
 382         csvLine[table.getIndex(WfoContentExportTable.RIGHTS_HOLDER)] = null;
 383
 384     }
 385
 386     private void handleMeasurementOrFact(WfoContentExportState state, String string, DescriptionElementBase deb,
 387             Taxon taxon) {
 388
 389     }
 390
 391     private void handleCommonName(WfoContentExportState state, CommonTaxonName commonName, Taxon taxon) {
 392         WfoContentExportTable table = WfoContentExportTable.VERNACULAR_NAME;
 393         //TODO i18n
 394         List<Language> languages = null;
 395         try {
 396             if (commonName instanceof CommonTaxonName) {
 397                 String[] csvLine = new String[table.getSize()];
 398 //                Distribution distribution = (Distribution) element;
 399 //                distributions.add(distribution);
 400
 401                 csvLine[table.getIndex(WfoContentExportTable.CN_VERNACULAR_NAME)] = commonName.getName();
 402
 403                 //language
 404                 if (commonName.getLanguage() != null) {
 405                     csvLine[table.getIndex(WfoContentExportTable.CN_VERNACULAR_NAME)] = commonName.getLanguage().getPreferredLabel(languages);
 406                 }
 407
 408                 //countryCode
 409                 NamedArea area = commonName.getArea();
 410                 if (area != null && area.getVocabulary() != null && area.getVocabulary().getUuid().equals(NamedArea.uuidCountryVocabulary)) {
 411                     String countryCode = ((Country)area).getIso3166_A2();
 412                     csvLine[table.getIndex(WfoContentExportTable.CN_COUNTRY_CODE)] = countryCode;
 413                 }
 414
 415                 //rights
 416                 handleRights(state, commonName, csvLine, table, taxon);
 417
 418                 //license
 419                 handleLicense(state, commonName, csvLine, table, taxon);
 420
 421                 //source
 422                 handleSource(state, commonName, table);
 423
 424                 state.getProcessor().put(table, commonName, csvLine);
 425             } else {
 426                 //TODO 1 is this handled elsewhere?
 427                 state.getResult()
 428                         .addError("The common name for the taxon " + taxon.getUuid()
 429                                 + " is not of type CommonTaxonName. Could not be exported. UUID of the common name: "
 430                                 + commonName.getUuid());
 431             }
 432         } catch (Exception e) {
 433             state.getResult().addException(e, "An unexpected error occurred when handling single common name "
 434                     + cdmBaseStr(commonName) + ": " + e.getMessage());
 435         }
 436     }
 437
 438     private void handleLicense(WfoContentExportState state, CommonTaxonName commonName,
 439             String[] csvLine, WfoContentExportTable table, Taxon taxon) {
 440         // TODO 1 handle License
 441
 442     }
 443
 444     private void handleRights(WfoContentExportState state, CommonTaxonName commonName,
 445             String[] csvLine, WfoContentExportTable table, Taxon taxon) {
 446         // TODO 1 handle rights
 447     }
 448
 449     private void handleDistribution(WfoContentExportState state, Distribution distribution, Taxon taxon) {
 450         WfoContentExportTable table = WfoContentExportTable.DISTRIBUTION;
 451         //TODO i18n
 452         List<Language> languages = null;
 453             try {
 454                 if (distribution instanceof Distribution) {
 455                     String[] csvLine = new String[table.getSize()];
 456 //                    Distribution distribution = (Distribution) element;
 457 //                    distributions.add(distribution);
 458                     NamedArea area = distribution.getArea();
 459
 460                     csvLine[table.getIndex(WfoContentExportTable.DIST_LOCALITY)] = area.getPreferredLabel(languages);
 461
 462                     //TDWG area
 463                     if (area.getVocabulary() !=null && area.getVocabulary().getUuid().equals(NamedArea.uuidTdwgAreaVocabulary)) {
 464                         String tdwgCode = area.getIdInVocabulary();
 465                         csvLine[table.getIndex(WfoContentExportTable.DIST_LOCATION_ID)] = tdwgCode;
 466                     }
 467
 468                     //countryCode
 469                     if (area.getVocabulary() !=null && area.getVocabulary().getUuid().equals(NamedArea.uuidCountryVocabulary)) {
 470                         String countryCode = ((Country)area).getIso3166_A2();
 471                         csvLine[table.getIndex(WfoContentExportTable.DIST_COUNTRY_CODE)] = countryCode;
 472                     }
 473
 474                     if (distribution.getStatus() != null) {
 475                         PresenceAbsenceTerm status = distribution.getStatus();
 476                         csvLine[table.getIndex(WfoContentExportTable.DIST_ESTABLISHMENT_MEANS)] = status.getPreferredLabel(languages);
 477                     }
 478                     //source
 479                     handleSource(state, distribution, table);
 480
 481                     //occurrencRemarks
 482                     //TODO 5 occurrence remarks correct?
 483                     csvLine[table.getIndex(WfoContentExportTable.DIST_OCCURRENCE_REMARKS)] = createAnnotationsString(distribution.getAnnotations());
 484
 485                     state.getProcessor().put(table, distribution, csvLine);
 486                 } else {
 487                     //TODO 1 is this handled elsewhere?
 488                     state.getResult()
 489                             .addError("The distribution description for the taxon " + taxon.getUuid()
 490                                     + " is not of type distribution. Could not be exported. UUID of the description element: "
 491                                     + distribution.getUuid());
 492                 }
 493             } catch (Exception e) {
 494                 state.getResult().addException(e, "An unexpected error occurred when handling single distribution "
 495                         + cdmBaseStr(distribution) + ": " + e.getMessage());
 496             }
 497     }
 498
 499     private boolean isUrl(String url) {
 500         try {
 501             if (url.startsWith("http")) {
 502                 URI.fromString(url);
 503                 return true;
 504             }
 505         } catch (Exception e) {
 506             //exception should return false
 507         }
 508         return false;
 509     }
 510
 511     private String toIsoDate(TimePeriod mediaCreated) {
 512         //TODO 2 date, what if end or freetext exist?
 513         Partial partial = mediaCreated.getStart();
 514         if (partial == null || !partial.isSupported(DateTimeFieldType.year())
 515                 || !partial.isSupported(DateTimeFieldType.monthOfYear()) && partial.isSupported(DateTimeFieldType.dayOfMonth())) {
 516             //TODO 2 date, log warning, also if mediaCreated.getEnd() != null or so
 517             return null;
 518         } else {
 519             DateTimeFormatter formatter = new DateTimeFormatterBuilder()
 520                     .appendYear(4, 4).appendLiteral('-')
 521                     .appendMonthOfYear(2).appendLiteral('-')
 522                     .appendDayOfMonth(2)
 523                     .toFormatter();
 524             return partial.toString(formatter);
 525         }
 526     }
 527
 528     /**
 529      * transforms the given date to an iso date
 530      */
 531     protected String toIsoDate(DateTime dateTime) {
 532         if (dateTime == null) {
 533             return null;
 534         }
 535         DateTimeFormatter formatter = new DateTimeFormatterBuilder()
 536                 .appendYear(4, 4).appendLiteral('-')
 537                 .appendMonthOfYear(2).appendLiteral('-')
 538                 .appendDayOfMonth(2)
 539                 .toFormatter();
 540         return formatter.print(dateTime);
 541     }
 542
 543     private String getRemarks(AnnotatableEntity entity) {
 544         String remarks = null;
 545         for (Annotation a : entity.getAnnotations()) {
 546             //TODO 3 handle other annotation types
 547             if (AnnotationType.EDITORIAL().equals(a.getAnnotationType())
 548                     && CdmUtils.isNotBlank(a.getText())){
 549                 remarks = CdmUtils.concat(";", remarks, a.getText());
 550             }
 551         }
 552         return remarks;
 553     }
 554
 555 //    private void handleMetaData(ColDpExportState state) {
 556 //        ColDpExportTable table = ColDpExportTable.METADATA;
 557 //        String[] csvLine = new String[table.getSize()];
 558 ////        csvLine[table.getIndex(CdmLightExportTable.INSTANCE_ID)] = state.getConfig().getInctanceId();
 559 ////        csvLine[table.getIndex(CdmLightExportTable.INSTANCE_NAME)] = state.getConfig().getInstanceName();
 560 //        csvLine[table.getIndex(ColDpExportTable.DATASET_BASE_URL)] = state.getConfig().getBase_url();
 561 //        csvLine[table.getIndex(ColDpExportTable.DATASET_CONTRIBUTOR)] = state.getConfig().getContributor();
 562 //        csvLine[table.getIndex(ColDpExportTable.DATASET_CREATOR)] = state.getConfig().getCreator();
 563 //        csvLine[table.getIndex(ColDpExportTable.DATASET_DESCRIPTION)] = state.getConfig().getDescription();
 564 //        csvLine[table.getIndex(ColDpExportTable.DATASET_DOWNLOAD_LINK)] = state.getConfig().getDataset_download_link();
 565 //        csvLine[table.getIndex(ColDpExportTable.DATASET_KEYWORDS)] = state.getConfig().getKeywords();
 566 //        csvLine[table.getIndex(ColDpExportTable.DATASET_LANDINGPAGE)] = state.getConfig().getDataSet_landing_page();
 567 //
 568 //        csvLine[table.getIndex(ColDpExportTable.DATASET_LANGUAGE)] = state.getConfig().getLanguage() != null? state.getConfig().getLanguage().getLabel(): null;
 569 //        csvLine[table.getIndex(ColDpExportTable.DATASET_LICENCE)] = state.getConfig().getLicence();
 570 //        csvLine[table.getIndex(ColDpExportTable.DATASET_LOCATION)] = state.getConfig().getLocation();
 571 //        csvLine[table.getIndex(ColDpExportTable.DATASET_RECOMMENDED_CITATTION)] = state.getConfig().getRecommended_citation();
 572 //        csvLine[table.getIndex(ColDpExportTable.DATASET_TITLE)] = state.getConfig().getTitle();
 573 //        state.getProcessor().put(table, "", csvLine);
 574 //    }
 575
 576     private String createMultilanguageString(Map<Language, LanguageString> multilanguageText) {
 577         String text = "";
 578         int index = multilanguageText.size();
 579         for (LanguageString langString : multilanguageText.values()) {
 580             text += langString.getText();
 581             if (index > 1) {
 582                 text += "; ";
 583             }
 584             index--;
 585         }
 586         return text;
 587     }
 588
 589     private String createAnnotationsString(Set<Annotation> annotations) {
 590         StringBuffer strBuff = new StringBuffer();
 591
 592         for (Annotation ann : annotations) {
 593             if (ann.getAnnotationType() == null || !ann.getAnnotationType().equals(AnnotationType.TECHNICAL())) {
 594                 strBuff.append(ann.getText());
 595                 strBuff.append("; ");
 596             }
 597         }
 598
 599         if (strBuff.length() > 2) {
 600             return strBuff.substring(0, strBuff.length() - 2);
 601         } else {
 602             return null;
 603         }
 604     }
 605
 606     private void handleSource(WfoContentExportState state, DescriptionElementBase element,
 607             WfoContentExportTable factsTable) {
 608 //        ColDpExportTable table = ColDpExportTable.FACT_SOURCES;
 609 //        try {
 610 //            Set<DescriptionElementSource> sources = element.getSources();
 611 //
 612 //            for (DescriptionElementSource source : sources) {
 613 //                if (!(source.getType().equals(OriginalSourceType.Import)
 614 //                        && state.getConfig().isExcludeImportSources())) {
 615 //                    String[] csvLine = new String[table.getSize()];
 616 //                    Reference ref = source.getCitation();
 617 //                    if ((ref == null) && (source.getNameUsedInSource() == null)) {
 618 //                        continue;
 619 //                    }
 620 //                    if (ref != null) {
 621 //                        if (!state.getReferenceStore().contains(ref.getUuid())) {
 622 //                            handleReference(state, ref);
 623 //
 624 //                        }
 625 //                        csvLine[table.getIndex(ColDpExportTable.REFERENCE_FK)] = getId(state, ref);
 626 //                    }
 627 //                    csvLine[table.getIndex(ColDpExportTable.FACT_FK)] = getId(state, element);
 628 //
 629 //                    csvLine[table.getIndex(ColDpExportTable.NAME_IN_SOURCE_FK)] = getId(state,
 630 //                            source.getNameUsedInSource());
 631 //                    csvLine[table.getIndex(ColDpExportTable.FACT_TYPE)] = factsTable.getTableName();
 632 //                    if (StringUtils.isBlank(csvLine[table.getIndex(ColDpExportTable.REFERENCE_FK)])
 633 //                            && StringUtils.isBlank(csvLine[table.getIndex(ColDpExportTable.NAME_IN_SOURCE_FK)])) {
 634 //                        continue;
 635 //                    }
 636 //                    state.getProcessor().put(table, source, csvLine);
 637 //                }
 638 //            }
 639 //        } catch (Exception e) {
 640 //            state.getResult().addException(e, "An unexpected error occurred when handling single source "
 641 //                    + cdmBaseStr(element) + ": " + e.getMessage());
 642 //        }
 643     }
 644
 645     private String getTitleCache(IIdentifiableEntity identEntity) {
 646         if (identEntity == null) {
 647             return "";
 648         }
 649         // TODO 3 titleCache refresh?
 650         return identEntity.getTitleCache();
 651     }
 652
 653     private String getId(WfoContentExportState state, ICdmBase cdmBase) {
 654         if (cdmBase == null) {
 655             return "";
 656         }
 657         // TODO 4 id type, make configurable
 658         return cdmBase.getUuid().toString();
 659     }
 660
 661     private String handleName(WfoContentExportState state, WfoContentExportTable table, String[] csvLine,
 662             TaxonName name) {
 663
 664         name = CdmBase.deproxy(name);
 665         if (name == null || state.getNameStore().containsKey(name.getId())) {
 666             if (name == null) {
 667                 state.getResult().addError("No name was given for taxon.", "handleName");
 668                 state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 669             }
 670             return null;
 671         }
 672
 673         String wfoId = null;
 674         try {
 675
 676             state.getNameStore().put(name.getId(), name.getUuid());
 677
 678             //taxonID
 679             wfoId = getWfoId(state, name, true);
 680             if (isBlank(wfoId)) {
 681                 String message = "No WFO-ID given for taxon " + name.getTitleCache() + ". Taxon ignored";
 682                 state.getResult().addError(message);
 683                 state.getResult().setState(ExportResultState.INCOMPLETE_WITH_ERROR);
 684                 return null;
 685             }else {
 686                 csvLine[table.getIndex(WfoContentExportTable.TAXON_ID)] = wfoId;
 687             }
 688
 689             //scientificNameID
 690             //TODO 9 add IPNI ID if exists
 691             csvLine[table.getIndex(WfoContentExportTable.NAME_SCIENTIFIC_NAME_ID)] = null;
 692
 693             //scientificName
 694             if (name.isProtectedTitleCache()) {
 695                 //TODO 7 make it configurable if we should always take titleCache if titleCache is protected, as nameCache may not necessarily
 696                 //     have complete data if titleCache is protected as it is considered to be irrelevant or at least preliminary
 697                 String message = "";
 698                 if (StringUtils.isNotEmpty(name.getNameCache())) {
 699                     csvLine[table.getIndex(WfoContentExportTable.NAME_SCIENTIFIC_NAME)] = name.getNameCache();
 700                     message = "ScientificName: Name cache " + name.getNameCache() + " used for name with protected titleCache " +  name.getTitleCache();
 701                 }else {
 702                     csvLine[table.getIndex(WfoContentExportTable.NAME_SCIENTIFIC_NAME)] = name.getTitleCache();
 703                     message = "ScientificName: Name has protected titleCache and no explicit nameCache: " +  name.getTitleCache();
 704                 }
 705                 state.getResult().addWarning(message);  //TODO 7 add location to warning
 706             } else {
 707                 csvLine[table.getIndex(WfoContentExportTable.NAME_SCIENTIFIC_NAME)] = name.getNameCache();
 708             }
 709
 710             //rank
 711             Rank rank = name.getRank();
 712             String rankStr = state.getTransformer().getCacheByRank(rank);
 713             if (rankStr == null) {
 714                 String message = rank == null ? "No rank" : ("Rank not supported by WFO:" + rank.getLabel())
 715                         + "Taxon not handled in export: " + name.getTitleCache();
 716                 state.getResult().addWarning(message);  //TODO 2 warning sufficient for missing rank? + location
 717                 return wfoId;
 718             }
 719             csvLine[table.getIndex(WfoContentExportTable.RANK)] = rankStr;
 720
 721             //scientificNameAuthorship
 722             //TODO 3 handle empty authorship cache warning
 723             csvLine[table.getIndex(WfoContentExportTable.NAME_AUTHORSHIP)] = name.getAuthorshipCache();
 724
 725             //family
 726             //TODO 2 family handling
 727             csvLine[table.getIndex(WfoContentExportTable.TAX_FAMILY)] = state.getFamilyStr();
 728
 729             //name parts
 730             csvLine[table.getIndex(WfoContentExportTable.TAX_GENUS)] = name.isSupraGeneric()? null : name.getGenusOrUninomial();
 731             csvLine[table.getIndex(WfoContentExportTable.NAME_SPECIFIC_EPITHET)] = name.getSpecificEpithet();
 732             csvLine[table.getIndex(WfoContentExportTable.NAME_INFRASPECIFIC_EPITHET)] = name.getInfraSpecificEpithet();
 733
 734             //nom. ref
 735             String nomRef = NomenclaturalSourceFormatter.INSTANCE().format(name.getNomenclaturalSource());
 736             csvLine[table.getIndex(WfoContentExportTable.NAME_PUBLISHED_IN)] = nomRef;
 737
 738         } catch (Exception e) {
 739             state.getResult().addException(e,
 740                     "An unexpected error occurred when handling the name " + cdmBaseStr(name) + ": " + name.getTitleCache() + ": " + e.getMessage());
 741
 742             e.printStackTrace();
 743         }
 744
 745         return wfoId;
 746     }
 747
 748     private String getWfoId(WfoContentExportState state, TaxonName name, boolean warnIfNotExists) {
 749         Identifier wfoId = name.getIdentifier(IdentifierType.uuidWfoNameIdentifier);
 750         if (wfoId == null && warnIfNotExists) {
 751             String message = "No wfo-id given for name: " + name.getTitleCache()+"/"+ name.getUuid();
 752             state.getResult().addWarning(message);  //TODO 5 data location
 753         }
 754         return wfoId == null ? null : wfoId.getIdentifier();
 755     }
 756
 757     private String makeNameStatus(WfoContentExportState state, TaxonName name) {
 758         try {
 759
 760             //TODO 1 what is with dubium
 761             if (name.isLegitimate()) {
 762                 if (name.isConserved()) {
 763                     return "Conserved";
 764                 }else {
 765                     return "Valid";
 766                 }
 767             } else if (name.isRejected()) {
 768                 return "Rejected";
 769             } else if (name.isIllegitimate()) {
 770                 return "Illegitimate";
 771             } else if (name.isInvalid()) {
 772                 //TODO 2 handle original spellings for name status
 773                 if (name.isOrthographicVariant()) {
 774                     return "orthografia";
 775                 }else {
 776                     return "invalid";
 777                 }
 778             } else {
 779                 String message = "Unhandled name status case for name: " + name.getTitleCache() +
 780                         ". Status not handled correctly.";
 781                 state.getResult().addWarning(message);
 782                 return "undefined";
 783             }
 784         } catch (Exception e) {
 785             state.getResult().addException(e, "An unexpected error occurred when extracting status string for "
 786                     + cdmBaseStr(name) + ": " + e.getMessage());
 787             return "";
 788         }
 789     }
 790
 791     private void handleHomotypicalGroup(WfoContentExportState state, HomotypicalGroup group, Taxon acceptedTaxon) {
 792         try {
 793
 794             List<TaxonName> typifiedNames = new ArrayList<>();
 795             if (acceptedTaxon != null){
 796                 List<Synonym> synonymsInGroup = acceptedTaxon.getSynonymsInGroup(group);
 797                 if (group.equals(acceptedTaxon.getHomotypicGroup())){
 798                     typifiedNames.add(acceptedTaxon.getName());
 799                 }
 800                 synonymsInGroup.stream().forEach(synonym -> typifiedNames.add(CdmBase.deproxy(synonym.getName())));
 801             }
 802
 803
 804             TaxonName firstname = null;
 805             for (TaxonName name: typifiedNames){
 806                 Iterator<Taxon> taxa = name.getTaxa().iterator();
 807                 while(taxa.hasNext()){
 808                     Taxon taxon = taxa.next();
 809                     if(!(taxon.isMisapplication() || taxon.isProparteSynonym())){
 810                         firstname = name;
 811                         break;
 812                     }
 813                 }
 814             }
 815
 816         } catch (Exception e) {
 817             state.getResult().addException(e, "An unexpected error occurred when handling homotypic group "
 818                     + cdmBaseStr(group) + ": " + e.getMessage());
 819         }
 820     }
 821
 822     private void handleReference(WfoContentExportState state, Reference reference) {
 823         try {
 824             if (state.getReferenceStore().contains(reference.getUuid())) {
 825                 return;
 826             }
 827             reference = CdmBase.deproxy(reference);
 828
 829             state.addReferenceToStore(reference);
 830             WfoContentExportTable table = WfoContentExportTable.REFERENCE;
 831             String[] csvLine = new String[table.getSize()];
 832
 833             csvLine[table.getIndex(WfoContentExportTable.IDENTIFIER)] = getId(state, reference);
 834
 835             //TODO 2 correct?, ref biblio citation
 836             csvLine[table.getIndex(WfoContentExportTable.REF_BIBLIO_CITATION)] = reference.getCitation();
 837
 838             //TODO 1 uri (doi, uri or ext_link
 839 //            csvLine[table.getIndex(WfoContentExportTable.REF_DOI)] = reference.getDoiString();
 840 //
 841 //            //TODO 2 reference link link (=> external link)
 842 ////            csvLine[table.getIndex(ColDpExportTable.LINK)] = null;
 843 //            if (reference.getUri() != null) {
 844 //                csvLine[table.getIndex(WfoContentExportTable.LINK)] = reference.getUri().toString();
 845 //            }
 846
 847             state.getProcessor().put(table, reference, csvLine);
 848         } catch (Exception e) {
 849             e.printStackTrace();
 850             state.getResult().addException(e, "An unexpected error occurred when handling reference "
 851                     + cdmBaseStr(reference) + ": " + e.getMessage());
 852         }
 853     }
 854
 855     /**
 856      * Returns a string representation of the {@link CdmBase cdmBase} object for
 857      * result messages.
 858      */
 859     private String cdmBaseStr(CdmBase cdmBase) {
 860         if (cdmBase == null) {
 861             return "-no object available-";
 862         } else {
 863             return cdmBase.getClass().getSimpleName() + ": " + cdmBase.getUuid();
 864         }
 865     }
 866
 867     @Override
 868     protected boolean doCheck(WfoContentExportState state) {
 869         return false;
 870     }
 871
 872     @Override
 873     protected boolean isIgnore(WfoContentExportState state) {
 874         return false;
 875     }
 876 }