cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/csv/redlist/demo/CsvDemoExport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.csv.redlist.demo;
  11
  12 import java.io.ByteArrayOutputStream;
  13 import java.io.PrintWriter;
  14 import java.util.ArrayList;
  15 import java.util.Collections;
  16 import java.util.Comparator;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Set;
  20 import java.util.UUID;
  21
  22 import org.apache.log4j.Logger;
  23 import org.springframework.stereotype.Component;
  24 import org.springframework.transaction.TransactionStatus;
  25
  26 import eu.etaxonomy.cdm.io.csv.redlist.out.CsvTaxExportStateRedlist;
  27 import eu.etaxonomy.cdm.model.common.CdmBase;
  28 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  29 import eu.etaxonomy.cdm.model.common.Language;
  30 import eu.etaxonomy.cdm.model.common.RelationshipTermBase;
  31 import eu.etaxonomy.cdm.model.description.CategoricalData;
  32 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  33 import eu.etaxonomy.cdm.model.description.Distribution;
  34 import eu.etaxonomy.cdm.model.description.Feature;
  35 import eu.etaxonomy.cdm.model.description.State;
  36 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  37 import eu.etaxonomy.cdm.model.description.TextData;
  38 import eu.etaxonomy.cdm.model.location.NamedArea;
  39 import eu.etaxonomy.cdm.model.name.NonViralName;
  40 import eu.etaxonomy.cdm.model.name.Rank;
  41 import eu.etaxonomy.cdm.model.reference.Reference;
  42 import eu.etaxonomy.cdm.model.taxon.Classification;
  43 import eu.etaxonomy.cdm.model.taxon.Synonym;
  44 import eu.etaxonomy.cdm.model.taxon.SynonymRelationship;
  45 import eu.etaxonomy.cdm.model.taxon.SynonymRelationshipType;
  46 import eu.etaxonomy.cdm.model.taxon.Taxon;
  47 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  48 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  49 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  50 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  51
  52
  53 /**
  54  * @author a.oppermann
  55  * @created 18.10.2012
  56  */
  57
  58 @Component
  59 public class CsvDemoExport extends CsvDemoBase {
  60         private static final Logger logger = Logger.getLogger(CsvDemoExport.class);
  61
  62         private static final String ROW_TYPE = "http://rs.tdwg.org/dwc/terms/Taxon";
  63         private static final String fileName = "RedlistCoreTax.csv";
  64
  65         public CsvDemoExport() {
  66                 super();
  67                 this.ioName = this.getClass().getSimpleName();
  68         }
  69
  70
  71         /** Retrieves data from a CDM DB and serializes them CDM to CSV.
  72          * Starts with root taxa and traverses the classification to retrieve
  73          * children taxa, synonyms, relationships, descriptive data, red list
  74          * status (features).
  75          * Taxa that are not part of the classification are not found.
  76          *
  77          * @param exImpConfig
  78          * @param dbname
  79          * @param filename
  80          */
  81         @Override
  82         protected void doInvoke(CsvDemoExportState state){
  83                 CsvDemoExportConfigurator config = state.getConfig();
  84                 TransactionStatus txStatus = startTransaction(true);
  85                 List<NamedArea> selectedAreas = config.getNamedAreas();
  86                 Set<Classification> classificationSet = assembleClassificationSet(config);
  87
  88                 PrintWriter writer = null;
  89                 ByteArrayOutputStream byteArrayOutputStream;
  90                 try {
  91                         byteArrayOutputStream = config.getByteArrayOutputStream();
  92                         writer = new PrintWriter(byteArrayOutputStream);
  93
  94                         //geographical Filter
  95                         List<TaxonNode> taxonNodes = handleGeographicalFilter(state, classificationSet);
  96
  97                         //sorting List
  98                         sortTaxonNodes(taxonNodes);
  99
 100                         for (TaxonNode node : taxonNodes){
 101                                 Taxon taxon = CdmBase.deproxy(node.getTaxon(), Taxon.class);
 102                                 CsvDemoRecord record = assembleRecord(state);
 103                                 NonViralName<?> name = CdmBase.deproxy(taxon.getName(), NonViralName.class);
 104                                 Classification classification = node.getClassification();
 105                                 config.setClassificationTitleCache(classification.getTitleCache());
 106                                 if (! this.recordExists(taxon)){
 107                                         handleTaxonBase(record, taxon, name, taxon, classification, null, false, false, config, node);
 108                                         record.write(writer);
 109                                         this.addExistingRecord(taxon);
 110                                 }
 111                                 //misapplied names
 112                                 handleMisapplication(taxon, writer, classification, record, config, node);
 113                                 writer.flush();
 114                         }
 115                 } catch (ClassCastException e) {
 116                         e.printStackTrace();
 117                 }
 118                 finally{
 119                         writer.close();
 120                         this.clearExistingRecordIds();
 121                 }
 122                 commitTransaction(txStatus);
 123                 return;
 124
 125         }
 126
 127
 128
 129         //TODO: Exception handling
 130         /**
 131          *
 132          * @param config
 133          * @return
 134          */
 135         protected Set<Classification> assembleClassificationSet(CsvDemoExportConfigurator config){
 136                 if(config != null){
 137                         Set<UUID> classificationUuidSet = config.getClassificationUuids();
 138                         List<Classification> classificationList = getClassificationService().find(classificationUuidSet);
 139                         Set<Classification> classificationSet = new HashSet<Classification>();
 140                         classificationSet.addAll(classificationList);
 141                         return classificationSet;
 142                 }
 143                 return null;
 144         }
 145
 146         //TODO: Exception handling
 147         /**
 148          *
 149          * @param state
 150          * @return
 151          */
 152         private CsvDemoRecord assembleRecord(CsvDemoExportState state) {
 153                 if(state!=null){
 154                         CsvDemoExportConfigurator config = state.getConfig();
 155                         CsvDemoMetaDataRecord metaRecord = new CsvDemoMetaDataRecord(true, fileName, ROW_TYPE);
 156                         state.addMetaRecord(metaRecord);
 157                         CsvDemoRecord record = new CsvDemoRecord(metaRecord, config);
 158                         return record;
 159                 }
 160                 return null;
 161         }
 162
 163         /**
 164          * Takes positive List of areas and iterates over a given classification
 165          * and their {@link Taxon} to return all {@link Taxon} with the desired
 166          * geographical attribute.
 167          *
 168          * <p><p>
 169          *
 170          * If selectedAreas is null all {@link TaxonNode}s of the given {@link Classification} will be returned.
 171          *
 172          * @param selectedAreas
 173          * @param classificationSet
 174          * @return
 175          */
 176         protected List<TaxonNode> handleGeographicalFilter(CsvDemoExportState state,
 177                         Set<Classification> classificationSet) {
 178                 List<TaxonNode> filteredNodes = new ArrayList<TaxonNode>();
 179                 List<TaxonNode> allNodes =  getAllNodes(classificationSet);
 180                 //Geographical filter
 181                 if(state.getConfig().isDoGeographicalFilter()){
 182                         List<NamedArea> selectedAreas = state.getConfig().getNamedAreas();
 183                         logger.info(selectedAreas.size());
 184                         if(selectedAreas != null && !selectedAreas.isEmpty() && selectedAreas.size() < 16){
 185                                 //                              if(selectedAreas.size() == 16){
 186                                 //                                      //Germany TDWG Level 3
 187                                 //                                      String germany="uu7b7c2db5-aa44-4302-bdec-6556fd74b0b9id";
 188                                 //                                      selectedAreas.add((NamedArea) getTermService().find(UUID.fromString(germany)));
 189                                 //                              }
 190                                 for (TaxonNode node : allNodes){
 191                                         Taxon taxon = CdmBase.deproxy(node.getTaxon(), Taxon.class);
 192                                         Set<TaxonDescription> descriptions = taxon.getDescriptions();
 193                                         for (TaxonDescription description : descriptions){
 194                                                 for (DescriptionElementBase el : description.getElements()){
 195                                                         if (el.isInstanceOf(Distribution.class) ){
 196                                                                 Distribution distribution = CdmBase.deproxy(el, Distribution.class);
 197                                                                 NamedArea area = distribution.getArea();
 198                                                                 for(NamedArea selectedArea:selectedAreas){
 199                                                                         if(selectedArea.getUuid().equals(area.getUuid())){
 200                                                                                 filteredNodes.add(node);
 201                                                                         }
 202                                                                 }
 203                                                         }
 204                                                 }
 205                                         }
 206                                 }
 207                         }else{
 208                                 filteredNodes = allNodes;
 209                         }
 210                 }
 211                 return filteredNodes;
 212         }
 213
 214         /**
 215          * handles misapplied {@link Taxon}
 216          * @param taxon
 217          * @param writer
 218          * @param classification
 219          * @param metaRecord
 220          * @param config
 221          */
 222         private void handleMisapplication(Taxon taxon, PrintWriter writer, Classification classification, CsvDemoRecord record, CsvDemoExportConfigurator config, TaxonNode node) {
 223                 Set<Taxon> misappliedNames = taxon.getMisappliedNames();
 224                 for (Taxon misappliedName : misappliedNames ){
 225 //                      CsvTaxRecordRedlist record = new CsvTaxRecordRedlist(metaRecord, config);
 226                         TaxonRelationshipType relType = TaxonRelationshipType.MISAPPLIED_NAME_FOR();
 227                         NonViralName<?> name = CdmBase.deproxy(misappliedName.getName(), NonViralName.class);
 228
 229                         if (! this.recordExists(misappliedName)){
 230                                 handleTaxonBase(record, misappliedName, name, taxon, classification, relType, false, false, config, node);
 231                                 record.write(writer);
 232                                 this.addExistingRecord(misappliedName);
 233                         }
 234                 }
 235         }
 236
 237         /**
 238          * handles the information record for the actual {@link Taxon} including {@link Classification classification}, Taxon Name, Taxon ID,
 239          * Taxon Status, Synonyms, {@link Feature features} data
 240          * @param record the concrete information record
 241          * @param taxonBase {@link Taxon}
 242          * @param name
 243          * @param acceptedTaxon
 244          * @param parent
 245          * @param basionym
 246          * @param isPartial
 247          * @param isProParte
 248          * @param config
 249          * @param node
 250          * @param type
 251          */
 252         private void handleTaxonBase(CsvDemoRecord record,TaxonBase<?> taxonBase,
 253                         NonViralName<?> name, Taxon acceptedTaxon, Classification classification,
 254                         RelationshipTermBase<?> relType, boolean isProParte, boolean isPartial,
 255                         CsvDemoExportConfigurator config, TaxonNode node) {
 256
 257                 Taxon taxon = (Taxon) taxonBase;
 258                 List<Feature> features = config.getFeatures();
 259                 record.setHeadLinePrinted(config.isHasHeaderLines());
 260                 if(config.isRedlistFeatures()){
 261                         if(features != null){
 262                                 record.setPrintFeatures(features);
 263                         }
 264                 }
 265                 config.setHasHeaderLines(false);
 266
 267                 if(config.isClassification()){
 268                         record.setDatasetName(classification.getTitleCache());
 269                 }
 270                 if(config.isTaxonName()){
 271                         record.setScientificName(name.getTitleCache());
 272                 }
 273                 if(config.isTaxonNameID()){
 274                         record.setScientificNameId(name.getUuid().toString());
 275                 }
 276                 if(config.isAuthor()){
 277                         String authorshipCache = name.getAuthorshipCache();
 278                         if(authorshipCache == null){
 279                                 authorshipCache = "";
 280                         }
 281                         record.setAuthorName(authorshipCache);
 282                 }
 283                 if(config.isRank()){
 284                         String rank;
 285                         if(taxon.getName().getRank() == null){
 286                                 rank = "";
 287                         }else{
 288                                 rank = taxon.getName().getRank().toString();
 289                         }
 290                         record.setRank(rank);
 291                 }
 292                 if(config.isTaxonStatus()){
 293                         handleTaxonomicStatus(record, name, relType, isProParte, isPartial);
 294                 }
 295                 if(config.isAcceptedName()){
 296                         //TODO write routine for accepted Name
 297                 }
 298                 if(config.isTaxonConceptID()){
 299                         UUID taxonUuid = taxonBase.getUuid();
 300                         if(taxonUuid == null){
 301                                 taxonUuid = UUID.fromString("");
 302                         }
 303                         record.setTaxonConceptID(taxonUuid.toString());
 304                 }
 305                 if(config.isParentID()){
 306                         String parentUUID;
 307                         if(node.getParent().getTaxon() == null){
 308                                 parentUUID = "";
 309                         }else{
 310                                 parentUUID = node.getParent().getTaxon().getUuid().toString();
 311                         }
 312                         record.setParentUUID(parentUUID);
 313                 }
 314                 if(config.isLastChange()){
 315                         String lastChange;
 316                         if(taxon.getUpdated() == null){
 317                                 lastChange = "";
 318                         }else{
 319                                 lastChange = taxon.getUpdated().toString();
 320                         }
 321                         record.setLastChange(lastChange);
 322                 }
 323                 if(config.isSynonyms()){
 324                         handleSynonyms(record,taxon);
 325                 }
 326                 if(config.isDistributions()){
 327                         handleDiscriptionData(record, taxon);
 328                 }
 329                 if(config.isRedlistFeatures()){
 330                         if(features!= null) {
 331
 332                                 List<List<String>> featureCells = new ArrayList<List<String>>(features.size());
 333                                 for(int i = 0; i < features.size(); i++) {
 334                                         featureCells.add(new ArrayList<String>());
 335                                 }
 336                                 handleRelatedRedlistStatus(record, taxon, false, featureCells, features);
 337                                 handleRelatedRedlistStatus(record, taxon, true, featureCells, features);
 338
 339                         }
 340                 }
 341
 342                 if(config.isExternalID()){
 343                         Set<IdentifiableSource> sources = taxonBase.getSources();
 344                         for(IdentifiableSource source:sources){
 345                                 Reference<?> citation = source.getCitation();
 346                                 /*
 347                                  * TODO: handle this more generic.
 348                                  * see ticket #4040
 349                                  *
 350                                  */
 351                                 if(citation.getId() == 22){
 352                                         String idInSource = source.getIdInSource();
 353                                         if(idInSource == null){
 354                                                 idInSource = "";
 355                                         }
 356                                         record.setExternalID(idInSource);
 357
 358                                 }
 359                         }
 360                 }
 361         }
 362
 363         /**
 364          * @param record
 365          * @param name
 366          * @param type
 367          * @param isPartial
 368          * @param isProParte
 369          */
 370         private void handleTaxonomicStatus(
 371                         CsvDemoRecord record,
 372                         NonViralName<?> name,
 373                         RelationshipTermBase<?> type,
 374                         boolean isProParte,
 375                         boolean isPartial) {
 376                 if (type == null){
 377                         record.setTaxonomicStatus(name.getNomenclaturalCode().acceptedTaxonStatusLabel());
 378                 }else{
 379                         String status = name.getNomenclaturalCode().synonymStatusLabel();
 380                         if (type.equals(SynonymRelationshipType.HETEROTYPIC_SYNONYM_OF())){
 381                                 status = "heterotypicSynonym";
 382                         }else if(type.equals(SynonymRelationshipType.HOMOTYPIC_SYNONYM_OF())){
 383                                 status = "homotypicSynonym";
 384                         }else if(type.equals(TaxonRelationshipType.MISAPPLIED_NAME_FOR())){
 385                                 status = "misapplied";
 386                         }
 387                         if (isProParte){
 388                                 status = "proParteSynonym";
 389                         }else if (isPartial){
 390                                 String message = "Partial synonym is not part of the gbif toxonomic status vocabulary";
 391                                 logger.warn(message);
 392                                 status = "partialSynonym";
 393                         }
 394
 395                         record.setTaxonomicStatus(status);
 396                 }
 397         }
 398
 399         /**
 400          *
 401          * This method concatenates several synonyms in a list.
 402          *
 403          * @param record
 404          * @param taxon
 405          */
 406         private void handleSynonyms(CsvDemoRecord record, Taxon taxon) {
 407
 408                 Set<SynonymRelationship> synRels = taxon.getSynonymRelations();
 409                 ArrayList<String> synonyms = new ArrayList<String>();
 410                 for (SynonymRelationship synRel :synRels ){
 411                         Synonym synonym = synRel.getSynonym();
 412                         SynonymRelationshipType type = synRel.getType();
 413                         if (type == null){ // should not happen
 414                                 type = SynonymRelationshipType.SYNONYM_OF();
 415                         }
 416                         NonViralName<?> name = CdmBase.deproxy(synonym.getName(), NonViralName.class);
 417                         synonyms.add(name.getTitleCache());
 418                 }
 419                 record.setSynonyms(synonyms);
 420         }
 421
 422         /**
 423          *
 424          * @param record
 425          * @param taxon
 426          */
 427         private void handleDiscriptionData(CsvDemoRecord record, Taxon taxon) {
 428
 429                 Set<TaxonDescription> descriptions = taxon.getDescriptions();
 430                 ArrayList<String> distributions = new ArrayList<String>();
 431                 for (TaxonDescription description : descriptions){
 432                         for (DescriptionElementBase el : description.getElements()){
 433                                 if (el.isInstanceOf(Distribution.class) ){
 434                                                 Distribution distribution = CdmBase.deproxy(el, Distribution.class);
 435                                                 NamedArea area = distribution.getArea();
 436                                                 distributions.add(area.getTitleCache());
 437                                 }
 438
 439                         }
 440                 }
 441                 record.setCountryCode(distributions);
 442         }
 443         /**
 444          *
 445          * @param record
 446          * @param taxon
 447          * @param featureCells
 448          * @param features
 449          */
 450         private void handleRedlistStatus(CsvDemoRecord record, Taxon taxon, List<List<String>> featureCells, List<Feature> features){
 451                 Set<TaxonDescription> descriptions = taxon.getDescriptions();
 452
 453                 for (TaxonDescription description : descriptions){
 454                         for (DescriptionElementBase el : description.getElements()){
 455                                 if(el.isInstanceOf(CategoricalData.class)){
 456                                         CategoricalData categoricalData = CdmBase.deproxy(el, CategoricalData.class);
 457                                         for(State state:categoricalData.getStatesOnly()){
 458                                                 Feature stateFeature = categoricalData.getFeature();
 459                                                 // find matching feature and put data into according cell
 460                                                 for(int i = 0; i < features.size(); i++) {
 461                                                         if(features.get(i).equals(stateFeature)){
 462                                                                 List<String> cell = featureCells.get(i);
 463                                                                 cell.add(state.toString());
 464                                                         }
 465                                                 }
 466                                         }
 467                                 }else if(el.isInstanceOf(TextData.class)){
 468                                         TextData textData = CdmBase.deproxy(el, TextData.class);
 469                                         Feature textFeature = textData.getFeature();
 470                                         // find matching feature and put data into according cell
 471                                         for(int i = 0; i < features.size(); i++) {
 472                                                 if(features.get(i).equals(textFeature)){
 473                                                         List<String> cell = featureCells.get(i);
 474                                                         String text = textData.getText(Language.GERMAN());
 475                                                         text = text.replaceAll(System.getProperty("line.separator"), "");
 476                                                         text = text.replaceAll("                            ", " ");
 477                                                         cell.add(text);
 478
 479                                                 }
 480                                         }
 481                                 }
 482                         }
 483                 }
 484                 record.setFeatures(featureCells);
 485         }
 486
 487         /**
 488          *
 489          * @param record
 490          * @param taxon
 491          * @param relationFrom
 492          * @param featureCells
 493          * @param features
 494          */
 495         private void handleRelatedRedlistStatus(CsvDemoRecord record, Taxon taxon, boolean relationFrom, List<List<String>> featureCells, List<Feature> features) {
 496
 497                 if (relationFrom)handleRedlistStatus(record, taxon, featureCells, features);
 498
 499
 500                 Set<TaxonRelationship> taxRels;
 501                 if(relationFrom){
 502                         taxRels = taxon.getRelationsFromThisTaxon();
 503                 }else{
 504                         taxRels = taxon.getRelationsToThisTaxon();
 505                 }
 506                 for (TaxonRelationship taxRel:taxRels){
 507                         if(taxRel.getType().equals(TaxonRelationshipType.CONGRUENT_TO())){
 508                                 Taxon relatedTaxon;
 509                                 if(relationFrom){
 510                                         relatedTaxon = taxRel.getToTaxon();
 511                                 }else{
 512                                         relatedTaxon = taxRel.getFromTaxon();
 513                                 }
 514                                 handleRedlistStatus(record, relatedTaxon, featureCells, features);
 515                         }
 516                 }
 517         }
 518
 519
 520
 521         /**
 522          *
 523          * @param taxonNodes
 524          */
 525         private void sortTaxonNodes(List<TaxonNode> taxonNodes) {
 526                 Collections.sort(taxonNodes, new Comparator<TaxonNode>() {
 527
 528                         @Override
 529                         public int compare(TaxonNode tn1, TaxonNode tn2) {
 530                                 Taxon taxon1 = tn1.getTaxon();
 531                                 Taxon taxon2 = tn2.getTaxon();
 532                                 if(taxon1 != null && taxon2 != null){
 533                                         return taxon1.getTitleCache().compareTo(taxon2.getTitleCache());
 534                                 }
 535                                 else{
 536                                         return 0;
 537                                 }
 538                         }
 539                 });
 540         }
 541
 542         @Override
 543         protected boolean doCheck(CsvDemoExportState state) {
 544                 boolean result = true;
 545                 logger.warn("No check implemented for " + this.ioName);
 546                 return result;
 547         }
 548
 549         @Override
 550         protected boolean isIgnore(CsvDemoExportState state) {
 551                 return ! state.getConfig().isDoTaxa();
 552         }
 553
 554 }