cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/portal/PortalDtoLoader.java

   1 /**
   2 * Copyright (C) 2023 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.api.service.portal;
  10
  11 import java.awt.Color;
  12 import java.time.LocalDateTime;
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.Collections;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22 import java.util.stream.Collectors;
  23
  24 import org.apache.commons.lang3.StringUtils;
  25 import org.apache.logging.log4j.LogManager;
  26 import org.apache.logging.log4j.Logger;
  27 import org.joda.time.DateTime;
  28
  29 import com.fasterxml.jackson.core.JsonProcessingException;
  30
  31 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  32 import eu.etaxonomy.cdm.api.dto.portal.AnnotatableDto;
  33 import eu.etaxonomy.cdm.api.dto.portal.AnnotationDto;
  34 import eu.etaxonomy.cdm.api.dto.portal.CdmBaseDto;
  35 import eu.etaxonomy.cdm.api.dto.portal.CommonNameDto;
  36 import eu.etaxonomy.cdm.api.dto.portal.ContainerDto;
  37 import eu.etaxonomy.cdm.api.dto.portal.DistributionDto;
  38 import eu.etaxonomy.cdm.api.dto.portal.DistributionInfoDto;
  39 import eu.etaxonomy.cdm.api.dto.portal.DistributionTreeDto;
  40 import eu.etaxonomy.cdm.api.dto.portal.FactDto;
  41 import eu.etaxonomy.cdm.api.dto.portal.FactDtoBase;
  42 import eu.etaxonomy.cdm.api.dto.portal.FeatureDto;
  43 import eu.etaxonomy.cdm.api.dto.portal.IFactDto;
  44 import eu.etaxonomy.cdm.api.dto.portal.IndividualsAssociationDto;
  45 import eu.etaxonomy.cdm.api.dto.portal.MarkerDto;
  46 import eu.etaxonomy.cdm.api.dto.portal.MessagesDto;
  47 import eu.etaxonomy.cdm.api.dto.portal.MessagesDto.MessageType;
  48 import eu.etaxonomy.cdm.api.dto.portal.NamedAreaDto;
  49 import eu.etaxonomy.cdm.api.dto.portal.SingleSourcedDto;
  50 import eu.etaxonomy.cdm.api.dto.portal.SourceDto;
  51 import eu.etaxonomy.cdm.api.dto.portal.SourcedDto;
  52 import eu.etaxonomy.cdm.api.dto.portal.TaxonBaseDto;
  53 import eu.etaxonomy.cdm.api.dto.portal.TaxonInteractionDto;
  54 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto;
  55 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.ConceptRelationDTO;
  56 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.HomotypicGroupDTO;
  57 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.KeyDTO;
  58 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.MediaDTO;
  59 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.MediaRepresentationDTO;
  60 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.NameRelationDTO;
  61 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.SpecimenDTO;
  62 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.TaxonNodeAgentsRelDTO;
  63 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.TaxonNodeDTO;
  64 import eu.etaxonomy.cdm.api.dto.portal.config.DistributionInfoConfiguration;
  65 import eu.etaxonomy.cdm.api.dto.portal.config.TaxonPageDtoConfiguration;
  66 import eu.etaxonomy.cdm.api.service.exception.TypeDesignationSetException;
  67 import eu.etaxonomy.cdm.api.service.geo.DistributionServiceUtilities;
  68 import eu.etaxonomy.cdm.api.service.geo.IDistributionService;
  69 import eu.etaxonomy.cdm.api.service.l10n.LocaleContext;
  70 import eu.etaxonomy.cdm.api.service.name.TypeDesignationSetContainer;
  71 import eu.etaxonomy.cdm.api.service.name.TypeDesignationSetFormatter;
  72 import eu.etaxonomy.cdm.api.service.pager.Pager;
  73 import eu.etaxonomy.cdm.common.CdmUtils;
  74 import eu.etaxonomy.cdm.common.TreeNode;
  75 import eu.etaxonomy.cdm.compare.taxon.TaxonComparator;
  76 import eu.etaxonomy.cdm.format.common.TypedLabel;
  77 import eu.etaxonomy.cdm.format.description.CategoricalDataFormatter;
  78 import eu.etaxonomy.cdm.format.description.QuantitativeDataFormatter;
  79 import eu.etaxonomy.cdm.format.description.distribution.CondensedDistributionConfiguration;
  80 import eu.etaxonomy.cdm.format.reference.OriginalSourceFormatter;
  81 import eu.etaxonomy.cdm.format.taxon.TaxonRelationshipFormatter;
  82 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  83 import eu.etaxonomy.cdm.model.common.Annotation;
  84 import eu.etaxonomy.cdm.model.common.AnnotationType;
  85 import eu.etaxonomy.cdm.model.common.CdmBase;
  86 import eu.etaxonomy.cdm.model.common.ICdmBase;
  87 import eu.etaxonomy.cdm.model.common.Language;
  88 import eu.etaxonomy.cdm.model.common.LanguageString;
  89 import eu.etaxonomy.cdm.model.common.Marker;
  90 import eu.etaxonomy.cdm.model.common.MarkerType;
  91 import eu.etaxonomy.cdm.model.common.MultilanguageTextHelper;
  92 import eu.etaxonomy.cdm.model.common.SingleSourcedEntityBase;
  93 import eu.etaxonomy.cdm.model.common.VersionableEntity;
  94 import eu.etaxonomy.cdm.model.description.CategoricalData;
  95 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  96 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  97 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  98 import eu.etaxonomy.cdm.model.description.Distribution;
  99 import eu.etaxonomy.cdm.model.description.Feature;
 100 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
 101 import eu.etaxonomy.cdm.model.description.PolytomousKey;
 102 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
 103 import eu.etaxonomy.cdm.model.description.QuantitativeData;
 104 import eu.etaxonomy.cdm.model.description.TaxonDescription;
 105 import eu.etaxonomy.cdm.model.description.TaxonInteraction;
 106 import eu.etaxonomy.cdm.model.description.TemporalData;
 107 import eu.etaxonomy.cdm.model.description.TextData;
 108 import eu.etaxonomy.cdm.model.location.NamedArea;
 109 import eu.etaxonomy.cdm.model.media.ImageFile;
 110 import eu.etaxonomy.cdm.model.media.Media;
 111 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
 112 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
 113 import eu.etaxonomy.cdm.model.name.HomotypicalGroup;
 114 import eu.etaxonomy.cdm.model.name.NameRelationship;
 115 import eu.etaxonomy.cdm.model.name.NameRelationshipType;
 116 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
 117 import eu.etaxonomy.cdm.model.name.TaxonName;
 118 import eu.etaxonomy.cdm.model.name.TypeDesignationBase;
 119 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
 120 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
 121 import eu.etaxonomy.cdm.model.reference.ISourceable;
 122 import eu.etaxonomy.cdm.model.reference.NamedSource;
 123 import eu.etaxonomy.cdm.model.reference.NamedSourceBase;
 124 import eu.etaxonomy.cdm.model.reference.OriginalSourceBase;
 125 import eu.etaxonomy.cdm.model.taxon.Classification;
 126 import eu.etaxonomy.cdm.model.taxon.Synonym;
 127 import eu.etaxonomy.cdm.model.taxon.Taxon;
 128 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
 129 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
 130 import eu.etaxonomy.cdm.model.taxon.TaxonNodeAgentRelation;
 131 import eu.etaxonomy.cdm.model.taxon.TaxonNodeStatus;
 132 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
 133 import eu.etaxonomy.cdm.model.term.Representation;
 134 import eu.etaxonomy.cdm.model.term.TermBase;
 135 import eu.etaxonomy.cdm.model.term.TermNode;
 136 import eu.etaxonomy.cdm.model.term.TermTree;
 137 import eu.etaxonomy.cdm.strategy.cache.TaggedCacheHelper;
 138 import eu.etaxonomy.cdm.strategy.cache.TaggedText;
 139 import eu.etaxonomy.cdm.strategy.cache.taxon.TaxonBaseDefaultCacheStrategy;
 140
 141 /**
 142  * Loads the portal dto from a taxon instance.
 143  * Maybe later also supports loading from persistence.
 144  *
 145  * @author a.mueller
 146  * @date 09.01.2023
 147  */
 148 public class PortalDtoLoader {
 149
 150     private static final Logger logger = LogManager.getLogger();
 151
 152     private ICdmRepository repository;
 153
 154     public PortalDtoLoader(ICdmRepository repository) {
 155         this.repository = repository;
 156     }
 157
 158     public TaxonPageDto load(Taxon taxon, TaxonPageDtoConfiguration config) {
 159         TaxonPageDto result = new TaxonPageDto();
 160
 161         TaxonName name = taxon.getName();
 162
 163         //load 1:1
 164         //TODO supplementalData for name
 165         loadBaseData(taxon, result);
 166         result.setLastUpdated(getLastUpdated(null, taxon));
 167         result.setNameLabel(name != null? name.getTitleCache() : "");
 168         result.setLabel(CdmUtils.Nz(taxon.getTitleCache()));
 169 //        result.setTypedTaxonLabel(getTypedTaxonLabel(taxon, config));
 170         result.setTaggedLabel(getTaggedTaxon(taxon, config));
 171         handleRelatedNames(taxon.getName(), result, config);
 172
 173         loadTaxonNodes(taxon, result, config);
 174         loadSynonyms(taxon, result, config);
 175         loadConceptRelations(taxon, result, config);
 176         loadFacts(taxon, result, config);
 177         loadMedia(taxon, result, config);
 178         loadSpecimens(taxon, result, config);
 179         loadKeys(taxon, result, config);
 180
 181         return result;
 182     }
 183
 184     private List<TaggedText> getTaggedTaxon(TaxonBase<?> taxon, TaxonPageDtoConfiguration config) {
 185 //        List<TypedLabel> result = new ArrayList<>();
 186         TaxonBaseDefaultCacheStrategy<TaxonBase<?>> formatter = new TaxonBaseDefaultCacheStrategy<>();
 187         List<TaggedText> tags = formatter.getTaggedTitle(taxon);
 188         return tags;
 189     }
 190
 191     private void loadKeys(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 192         ContainerDto<KeyDTO> container = new ContainerDto<>();
 193
 194         //TODO other key types, but type must not be null, otherwise NPE
 195         Pager<PolytomousKey> keys = repository.getIdentificationKeyService().findKeysConvering(taxon, PolytomousKey.class, null, null, null);
 196         for (PolytomousKey key : keys.getRecords()) {
 197             KeyDTO dto = new KeyDTO();
 198             loadBaseData(key, dto);
 199             dto.setLabel(key.getTitleCache());
 200             dto.setKeyClass(key.getClass().getSimpleName());
 201             container.addItem(dto);
 202         }
 203         if (container.getCount() > 0) {
 204             result.setKeys(container);
 205         }
 206     }
 207
 208     private void loadSpecimens(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 209         //TODO load specimen from multiple places
 210
 211         ContainerDto<SpecimenDTO> container = new ContainerDto<>();
 212
 213         List<SpecimenOrObservationBase<?>> specimens = new ArrayList<>();
 214         for (TaxonDescription taxonDescription : taxon.getDescriptions()) {
 215             if (taxonDescription.isImageGallery()) {
 216                 continue;
 217             }
 218             for (DescriptionElementBase el : taxonDescription.getElements()) {
 219                 if (el.isInstanceOf(IndividualsAssociation.class)) {
 220                     IndividualsAssociation indAss = CdmBase.deproxy(el, IndividualsAssociation.class);
 221                     SpecimenOrObservationBase<?> specimen = indAss.getAssociatedSpecimenOrObservation();
 222                     specimens.add(specimen);
 223                 }
 224             }
 225         }
 226         List<SpecimenOrObservationBase<?>> typeSpecimens = loadTypeSpecimen(taxon.getName(), config);
 227         specimens.addAll(typeSpecimens);
 228         for (TaxonName syn : taxon.getSynonymNames()) {
 229             typeSpecimens = loadTypeSpecimen(syn, config);
 230             specimens.addAll(typeSpecimens);
 231         }
 232
 233         for (SpecimenOrObservationBase<?> specimen : specimens) {
 234             SpecimenDTO dto = new SpecimenDTO();
 235             loadBaseData(specimen, dto);
 236             dto.setLabel(specimen.getTitleCache());
 237             container.addItem(dto);
 238         }
 239         if (container.getCount() > 0 ) {
 240             result.setSpecimens(container);
 241         }
 242     }
 243
 244     private List<SpecimenOrObservationBase<?>> loadTypeSpecimen(TaxonName name, TaxonPageDtoConfiguration config) {
 245         List<SpecimenOrObservationBase<?>> result = new ArrayList<>();
 246         for (SpecimenTypeDesignation desig: name.getSpecimenTypeDesignations()){
 247             DerivedUnit specimen = desig.getTypeSpecimen();
 248             if (specimen != null) {
 249                 result.add(specimen);
 250             }
 251         }
 252         return result;
 253     }
 254
 255     private void loadMedia(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 256
 257         ContainerDto<MediaDTO> container = new ContainerDto<TaxonPageDto.MediaDTO>();
 258
 259         List<Media> medias = new ArrayList<>();
 260         for (TaxonDescription taxonDescription : taxon.getDescriptions()) {
 261             if (!taxonDescription.isImageGallery()) {
 262                 continue;
 263             }
 264
 265             List<Media> newMedia = taxonDescription.getElements().stream()
 266                 .filter(el->el.isInstanceOf(TextData.class))
 267                 .map(el->CdmBase.deproxy(el, TextData.class))
 268                 .filter(td->true)
 269                 .flatMap(td->td.getMedia().stream())
 270                 .collect(Collectors.toList())
 271                 ;
 272             medias.addAll(newMedia);
 273         }
 274         //TODO collect media from elsewhere
 275         for (Media media : medias) {
 276             MediaDTO dto = new TaxonPageDto.MediaDTO();
 277             loadBaseData(media, dto);
 278             dto.setLabel(media.getTitleCache());
 279             ContainerDto<MediaRepresentationDTO> representations = new ContainerDto<>();
 280             for (MediaRepresentation rep : media.getRepresentations()) {
 281                 MediaRepresentationDTO repDto = new MediaRepresentationDTO();
 282                 loadBaseData(rep, dto);
 283                 repDto.setMimeType(rep.getMimeType());
 284                 repDto.setSuffix(rep.getSuffix());
 285                 if (!rep.getParts().isEmpty()) {
 286                     //TODO handle message if n(parts) > 1
 287                     MediaRepresentationPart part = rep.getParts().get(0);
 288                     repDto.setUri(part.getUri());
 289                     repDto.setClazz(part.getClass());
 290                     repDto.setSize(part.getSize());
 291                     if (part.isInstanceOf(ImageFile.class)) {
 292                         ImageFile image = CdmBase.deproxy(part, ImageFile.class);
 293                         repDto.setHeight(image.getHeight());
 294                         repDto.setWidth(image.getWidth());
 295                     }
 296                     //TODO AudioFile etc.
 297                 }
 298                 representations.addItem(repDto);
 299             }
 300             if (representations.getCount() > 0) {
 301                 dto.setRepresentations(representations);
 302             }
 303             //TODO load representation data
 304             container.addItem(dto);
 305         }
 306
 307         if (container.getCount() > 0) {
 308             result.setMedia(container);
 309         }
 310     }
 311
 312     private void loadTaxonNodes(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 313         ContainerDto<TaxonNodeDTO> container = new ContainerDto<TaxonPageDto.TaxonNodeDTO>();
 314         for (TaxonNode node : taxon.getTaxonNodes()) {
 315             TaxonNodeDTO dto = new TaxonNodeDTO();
 316             loadBaseData(node, dto);
 317             //classification
 318             Classification classification = node.getClassification();
 319             if (classification != null) {
 320                 dto.setClassificationUuid(node.getClassification().getUuid());
 321                 dto.setClassificationLabel(classification.getName().getText());
 322             }
 323             //TODO lang/locale
 324             Language language = Language.DEFAULT();
 325
 326             //status
 327             TaxonNodeStatus status = node.getStatus();
 328             if (status != null) {
 329                 dto.setStatus(status.getLabel(language));
 330             }
 331             //statusNote
 332             Map<Language, LanguageString> statusNote = node.getStatusNote();
 333             if (statusNote != null) {
 334                 //TODO handle fallback lang
 335                 LanguageString statusNoteStr = statusNote.get(language);
 336                 if (statusNoteStr == null && statusNote.size() > 0) {
 337                     statusNoteStr = statusNote.entrySet().iterator().next().getValue();
 338                 }
 339                 if (statusNoteStr != null) {
 340                     dto.setStatusNote(statusNoteStr.getText());
 341                 }
 342             }
 343             //agent relations
 344             Set<TaxonNodeAgentRelation> agents = node.getAgentRelations();
 345             if (!agents.isEmpty()) {
 346                 for (TaxonNodeAgentRelation rel : agents) {
 347                     TaxonNodeAgentsRelDTO agentDto = new TaxonNodeAgentsRelDTO();
 348                     loadBaseData(rel, agentDto);
 349
 350                     //TODO laod
 351                     if (rel.getAgent() != null) {
 352                         agentDto.setAgent(rel.getAgent().getFullTitle());
 353                         agentDto.setAgentUuid(rel.getAgent().getUuid());
 354                         //TODO compute preferred external link
 355                         agentDto.setAgentLink(null);
 356                     }
 357                     if (rel.getType() != null) {
 358                         agentDto.setType(rel.getType().getTitleCache());
 359                         agentDto.setTypeUuid(rel.getType().getUuid());
 360                     }
 361                     dto.addAgent(agentDto);
 362                 }
 363             }
 364             container.addItem(dto);
 365         }
 366         if (container.getCount() > 0) {
 367             result.setTaxonNodes(container);
 368         }
 369
 370     }
 371
 372
 373     private void loadSynonyms(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 374 //        List<HomotypicalGroup> homotypicGroups = taxon.getHomotypicSynonymyGroups();
 375
 376         TaxonComparator comparator = new TaxonComparator();
 377
 378         TaxonName name = taxon.getName();
 379
 380         //TODO depending on config add/remove accepted name
 381
 382         //TODO check publish flag
 383
 384         //homotypic synonyms
 385         List<Synonym> homotypicSynonmys = taxon.getHomotypicSynonymsByHomotypicGroup(comparator);
 386         TaxonPageDto.HomotypicGroupDTO homotypicGroupDto = new TaxonPageDto.HomotypicGroupDTO();
 387         if (!homotypicSynonmys.isEmpty()) {
 388             loadBaseData(name.getHomotypicalGroup(), homotypicGroupDto);
 389
 390             for (Synonym syn : homotypicSynonmys) {
 391                 loadSynonymsInGroup(homotypicGroupDto, syn, config);
 392             }
 393         }
 394         //TODO NPE
 395         handleTypification(name.getHomotypicalGroup(), homotypicGroupDto, result, config);
 396         result.setHomotypicSynonyms(homotypicGroupDto);
 397
 398         //heterotypic synonyms
 399         List<HomotypicalGroup> heteroGroups = taxon.getHeterotypicSynonymyGroups();
 400         if (heteroGroups.isEmpty()) {
 401             return;
 402         }
 403         ContainerDto<HomotypicGroupDTO> heteroContainer = new ContainerDto<>();
 404         result.setHeterotypicSynonymGroups(heteroContainer);
 405
 406         for (HomotypicalGroup hg : heteroGroups) {
 407             TaxonPageDto.HomotypicGroupDTO hgDto = new TaxonPageDto.HomotypicGroupDTO();
 408             loadBaseData(taxon.getName().getHomotypicalGroup(), hgDto);
 409             heteroContainer.addItem(hgDto);
 410
 411             List<Synonym> heteroSyns = taxon.getSynonymsInGroup(hg, comparator);
 412             for (Synonym syn : heteroSyns) {
 413                 loadSynonymsInGroup(hgDto, syn, config);
 414             }
 415             handleTypification(hg, hgDto, result, config);
 416         }
 417     }
 418
 419     private void handleTypification(HomotypicalGroup homotypicalGroup, HomotypicGroupDTO hgDto,
 420             TaxonPageDto result, TaxonPageDtoConfiguration config) {
 421
 422         boolean withCitation = true;
 423         boolean withStartingTypeLabel = true;
 424         boolean withNameIfAvailable = false;
 425         TypeDesignationSetFormatter formatter = new TypeDesignationSetFormatter(
 426                 withCitation, withStartingTypeLabel, withNameIfAvailable);
 427         Set<TypeDesignationBase<?>> desigs = homotypicalGroup.getTypeDesignations();
 428         try {
 429             TypeDesignationSetContainer manager = TypeDesignationSetContainer.NewDefaultInstance((Set)desigs);
 430             List<TaggedText> tags = formatter.toTaggedText(manager);
 431             String label = TaggedCacheHelper.createString(tags);
 432             hgDto.setTypes(label);
 433             hgDto.setTaggedTypes(tags);
 434 //            hgDto.setTypedTypes(null);
 435
 436         } catch (TypeDesignationSetException e) {
 437             result.addMessage(new MessagesDto(MessageType.ERROR, "Error when creating type designation information"));
 438         }
 439     }
 440
 441     private void loadConceptRelations(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 442
 443         //concept relations
 444         ContainerDto<ConceptRelationDTO> conceptRelContainer = new ContainerDto<>();
 445         TaxonRelationshipFormatter taxRelFormatter = TaxonRelationshipFormatter.INSTANCE();
 446
 447         //... MAN
 448         Set<TaxonRelationship> misappliedRels = taxon.getMisappliedNameRelations();
 449         for (TaxonRelationship rel : misappliedRels) {
 450             boolean inverse = true;
 451             boolean withoutName = false;
 452             loadConceptRelation(taxRelFormatter, rel, conceptRelContainer, inverse, withoutName);
 453         }
 454
 455         //... pro parte Synonyms
 456         Set<TaxonRelationship> proParteRels = taxon.getProParteAndPartialSynonymRelations();
 457         for (TaxonRelationship rel : proParteRels) {
 458             boolean inverse = true;
 459             boolean withoutName = false;
 460             loadConceptRelation(taxRelFormatter, rel, conceptRelContainer, inverse, withoutName);
 461         }
 462
 463         //TODO MAN and pp from this taxon
 464
 465         //... to-relations
 466         Set<TaxonRelationship> toRels = taxon.getRelationsToThisTaxon();
 467         for (TaxonRelationship rel : toRels) {
 468             boolean inverse = true;
 469             boolean withoutName = false;
 470             loadConceptRelation(taxRelFormatter, rel, conceptRelContainer, inverse, withoutName);
 471         }
 472
 473         //... from-relations
 474         Set<TaxonRelationship> fromRels = taxon.getRelationsFromThisTaxon();
 475         for (TaxonRelationship rel : fromRels) {
 476             boolean inverse = false;
 477             boolean withoutName = false;
 478             loadConceptRelation(taxRelFormatter, rel, conceptRelContainer, inverse, withoutName);
 479         }
 480
 481         if (conceptRelContainer.getCount() > 0) {
 482             result.setConceptRelations(conceptRelContainer);
 483         }
 484     }
 485
 486     private void loadConceptRelation(TaxonRelationshipFormatter taxRelFormatter, TaxonRelationship rel, ContainerDto<ConceptRelationDTO> conceptRelContainer, boolean inverse,
 487             boolean withoutName) {
 488         List<Language> languages = Arrays.asList(new Language[] {Language.DEFAULT()}); // TODO config.locales;
 489         List<TaggedText> tags = taxRelFormatter.getTaggedText(rel, inverse, languages, withoutName);
 490         String relLabel = TaggedCacheHelper.createString(tags);
 491         ConceptRelationDTO dto = new TaxonPageDto.ConceptRelationDTO();
 492         loadBaseData(rel, dto);
 493         Taxon relTaxon = inverse ? rel.getFromTaxon() : rel.getToTaxon();
 494         dto.setRelTaxonId(relTaxon.getId());
 495         dto.setRelTaxonUuid(relTaxon.getUuid());
 496         dto.setRelTaxonLabel(relTaxon.getTitleCache());
 497         dto.setLabel(relLabel);
 498         conceptRelContainer.addItem(dto);
 499     }
 500
 501     private void loadSynonymsInGroup(TaxonPageDto.HomotypicGroupDTO hgDto, Synonym syn, TaxonPageDtoConfiguration config) {
 502         TaxonBaseDto synDto = new TaxonBaseDto();
 503         loadBaseData(syn, synDto);
 504         synDto.setLabel(syn.getTitleCache());
 505         synDto.setTaggedLabel(getTaggedTaxon(syn, config));
 506         if (syn.getName() != null) {
 507             synDto.setNameLabel(syn.getName().getTitleCache());
 508             handleRelatedNames(syn.getName(), synDto, config);
 509         }
 510
 511         //TODO
 512         hgDto.addSynonym(synDto);
 513     }
 514
 515     private void handleRelatedNames(TaxonName name, TaxonBaseDto taxonDto, TaxonPageDtoConfiguration config) {
 516         //exclusions TODO handle via config
 517         Set<UUID> excludedTypes = new HashSet<>();  //both directions
 518         excludedTypes.add(NameRelationshipType.uuidBasionym);
 519         excludedTypes.add(NameRelationshipType.uuidReplacedSynonym);
 520         Set<UUID> excludedFromTypes = new HashSet<>(excludedTypes);
 521         Set<UUID> excludedToTypes = new HashSet<>(excludedTypes);
 522         //TODO non-types
 523
 524         //TODO config.getLocales();
 525         Language locale = Language.DEFAULT();
 526
 527         for (NameRelationship rel : name.getRelationsFromThisName()) {
 528             TaxonName relatedName = rel.getToName();
 529             if (relatedName == null || rel.getType() == null || excludedFromTypes.contains(rel.getType().getUuid())) {
 530                 continue;
 531             }
 532             NameRelationDTO dto = new NameRelationDTO();
 533             loadBaseData(rel, dto);
 534             //name
 535             dto.setNameUuid(relatedName.getUuid());
 536             dto.setNameLabel(relatedName.getTaggedName());
 537             //type
 538             dto.setRelTypeUuid(rel.getType().getUuid());
 539             Representation rep = rel.getType().getPreferredRepresentation(locale);
 540             dto.setRelType(rep == null ? rel.getType().toString() : rep.getLabel());
 541             //inverse
 542             dto.setInverse(false);
 543             taxonDto.addRelatedName(dto);
 544         }
 545
 546         //to relations
 547         for (NameRelationship rel : name.getRelationsToThisName()) {
 548             TaxonName relatedName = rel.getFromName();
 549             if (relatedName == null || rel.getType() == null || excludedFromTypes.contains(rel.getType().getUuid())) {
 550                 continue;
 551             }
 552             NameRelationDTO dto = new NameRelationDTO();
 553             loadBaseData(rel, dto);
 554             //name
 555             dto.setNameUuid(relatedName.getUuid());
 556             dto.setNameLabel(relatedName.getTaggedName());
 557             //type
 558             dto.setRelTypeUuid(rel.getType().getUuid());
 559             Representation rep = rel.getType().getPreferredInverseRepresentation(Arrays.asList(new Language[] {locale}));
 560             dto.setRelType(rep == null ? rel.getType().toString() : rep.getLabel());
 561             //inverse
 562             dto.setInverse(true);
 563             taxonDto.addRelatedName(dto);
 564         }
 565     }
 566
 567     private void loadFacts(Taxon taxon, TaxonPageDto taxonPageDto, TaxonPageDtoConfiguration config) {
 568
 569         //compute the features that do exist for this taxon
 570         Map<UUID, Feature> existingFeatureUuids = getExistingFeatureUuids(taxon);
 571
 572         //filter, sort and structure according to feature tree
 573         TreeNode<Feature, UUID> filteredRootNode;
 574         if (config.getFeatureTree() != null) {
 575
 576             //TODO class cast
 577             TermTree<Feature> featureTree = repository.getTermTreeService().find(config.getFeatureTree());
 578             filteredRootNode = filterFeatureNode(featureTree.getRoot(), existingFeatureUuids.keySet());
 579         } else {
 580             filteredRootNode = createDefaultFeatureNode(taxon);
 581         }
 582
 583         //load facts per feature
 584         Map<UUID,Set<DescriptionElementBase>> featureMap = loadFeatureMap(taxon);
 585
 586         //load final result
 587         if (!filteredRootNode.getChildren().isEmpty()) {
 588             ContainerDto<FeatureDto> features = new ContainerDto<>();
 589             for (TreeNode<Feature,UUID> node : filteredRootNode.getChildren()) {
 590                 handleFeatureNode(taxon, config, featureMap, features, node);
 591             }
 592             taxonPageDto.setFactualData(features);
 593         }
 594     }
 595
 596     private void handleFeatureNode(Taxon taxon, TaxonPageDtoConfiguration config,
 597             Map<UUID, Set<DescriptionElementBase>> featureMap, ContainerDto<FeatureDto> features,
 598             TreeNode<Feature, UUID> node) {
 599
 600         Feature feature = node.getData();
 601         //TODO locale
 602         FeatureDto featureDto = new FeatureDto(feature.getUuid(), feature.getId(), feature.getLabel());
 603         features.addItem(featureDto);
 604
 605         List<Distribution> distributions = new ArrayList<>();
 606
 607         //
 608         for (DescriptionElementBase fact : featureMap.get(feature.getUuid())){
 609             if (fact.isInstanceOf(Distribution.class)) {
 610                 distributions.add(CdmBase.deproxy(fact, Distribution.class));
 611             }else {
 612                 //TODO how to handle CommonNames, do we also want to have a data structure
 613                 //with Language|
 614 //                             -- Area|
 615 //                                    --name
 616                 // a bit like for distribution??
 617                 handleFact(featureDto, fact);
 618             }
 619         }
 620
 621         handleDistributions(config, featureDto, taxon, distributions);
 622         //TODO really needed?
 623         orderFacts(featureDto);
 624
 625         //children
 626         ContainerDto<FeatureDto> childFeatures = new ContainerDto<>();
 627         for (TreeNode<Feature,UUID> child : node.getChildren()) {
 628             handleFeatureNode(taxon, config, featureMap, childFeatures, child);
 629         }
 630         if (childFeatures.getCount() > 0) {
 631             featureDto.setSubFeatures(childFeatures);
 632         }
 633     }
 634
 635     //TODO not really used yet, only for distinguishing fact classes,
 636     //needs discussion if needed and how to implement.
 637     //we could also move compareTo methods to DTO classes but with this
 638     //remove from having only data in the DTO, no logic
 639     private void orderFacts(FeatureDto featureDto) {
 640         List<IFactDto> list = featureDto.getFacts().getItems();
 641         Collections.sort(list, (f1,f2)->{
 642             if (!f1.getClass().equals(f2.getClass())) {
 643                 return f1.getClass().getSimpleName().compareTo(f2.getClass().getSimpleName());
 644             }else {
 645                 if (f1 instanceof FactDto) {
 646                    FactDto fact1 = (FactDto)f1;
 647                    FactDto fact2 = (FactDto)f2;
 648
 649 //                   return fact1.getTypedLabel().toString().compareTo(fact2.getTypedLabel().toString());
 650                    return 0; //FIXME;
 651                 } else if (f1 instanceof CommonNameDto) {
 652                     int result = 0;
 653                     CommonNameDto fact1 = (CommonNameDto)f1;
 654                     CommonNameDto fact2 = (CommonNameDto)f2;
 655                     return 0;  //FIXME
 656                 }
 657             }
 658             return 0;  //FIXME
 659         });
 660
 661     }
 662
 663     private Map<UUID, Set<DescriptionElementBase>> loadFeatureMap(Taxon taxon) {
 664         Map<UUID, Set<DescriptionElementBase>> featureMap = new HashMap<>();
 665
 666         //... load facts
 667         for (TaxonDescription taxonDescription : taxon.getDescriptions()) {
 668             if (taxonDescription.isImageGallery()) {
 669                 continue;
 670             }
 671             for (DescriptionElementBase deb : taxonDescription.getElements()) {
 672                 Feature feature = deb.getFeature();
 673                 if (featureMap.get(feature.getUuid()) == null) {
 674                     featureMap.put(feature.getUuid(), new HashSet<>());
 675                 }
 676                 featureMap.get(feature.getUuid()).add(deb);
 677             }
 678         }
 679         return featureMap;
 680     }
 681
 682     private TreeNode<Feature, UUID> createDefaultFeatureNode(Taxon taxon) {
 683         TreeNode<Feature, UUID> root = new TreeNode<>();
 684         Set<Feature> requiredFeatures = new HashSet<>();
 685
 686         for (TaxonDescription taxonDescription : taxon.getDescriptions()) {
 687             if (taxonDescription.isImageGallery()) {
 688                 continue;
 689             }
 690             for (DescriptionElementBase deb : taxonDescription.getElements()) {
 691                 Feature feature = deb.getFeature();
 692                 if (feature != null) {  //null should not happen
 693                     requiredFeatures.add(feature);
 694                 }
 695             }
 696         }
 697         List<Feature> sortedChildren = new ArrayList<>(requiredFeatures);
 698         Collections.sort(sortedChildren, (f1,f2) -> f1.getTitleCache().compareTo(f2.getTitleCache()));
 699         sortedChildren.stream().forEachOrdered(f->root.addChild(new TreeNode<>(f.getUuid(), f)));
 700         return root;
 701     }
 702
 703     /**
 704      * Recursive call to a feature tree's feature node in order to creates a tree structure
 705      * ordered in the same way as the according feature tree but only containing features
 706      * that do really exist for the given taxon. If only a child node is required the parent
 707      * node/feature is also considered to be required.<BR>
 708      */
 709     private TreeNode<Feature, UUID> filterFeatureNode(TermNode<Feature> featureNode,
 710             Set<UUID> existingFeatureUuids) {
 711
 712         //first filter children
 713         List<TreeNode<Feature, UUID>> requiredChildNodes = new ArrayList<>();
 714         for (TermNode<Feature> childNode : featureNode.getChildNodes()) {
 715             TreeNode<Feature, UUID> child = filterFeatureNode(childNode, existingFeatureUuids);
 716             if (child != null) {
 717                 requiredChildNodes.add(child);
 718             }
 719         }
 720
 721         //if any child is required or this node is required ....
 722         if (!requiredChildNodes.isEmpty() ||
 723                 featureNode.getTerm() != null && existingFeatureUuids.contains(featureNode.getTerm().getUuid())) {
 724             TreeNode<Feature,UUID> result = new TreeNode<>();
 725             //add this nodes data
 726             Feature feature = featureNode.getTerm() == null ? null : featureNode.getTerm();
 727             if (feature != null) {
 728                 result.setNodeId(feature.getUuid());
 729                 result.setData(feature);
 730             }
 731             //add child data
 732             requiredChildNodes.stream().forEachOrdered(c->result.addChild(c));
 733             return result;
 734         }else {
 735             return null;
 736         }
 737     }
 738
 739     /**
 740      * Computes the (unsorted) set of features for  which facts exist
 741      * for the given taxon.
 742      */
 743     private Map<UUID, Feature> getExistingFeatureUuids(Taxon taxon) {
 744         Map<UUID, Feature> result = new HashMap<>();
 745         for (TaxonDescription taxonDescription : taxon.getDescriptions()) {
 746             if (taxonDescription.isImageGallery()) {
 747                 continue;
 748             }
 749             for (DescriptionElementBase deb : taxonDescription.getElements()) {
 750                 Feature feature = deb.getFeature();
 751                 if (feature != null) {  //null should not happen
 752                     result.put(feature.getUuid(), feature);
 753                 }
 754             }
 755         }
 756         return result;
 757     }
 758
 759     private void handleDistributions(TaxonPageDtoConfiguration config, FeatureDto featureDto,
 760             Taxon taxon, List<Distribution> distributions) {
 761
 762         if (distributions.isEmpty()) {
 763             return;
 764         }
 765         IDistributionService distributionService = repository.getDistributionService();
 766
 767         //configs
 768         DistributionInfoConfiguration distributionConfig = config.getDistributionInfoConfiguration();
 769         CondensedDistributionConfiguration condensedConfig = distributionConfig.getCondensedDistrConfig();
 770
 771         String statusColorsString = distributionConfig.getStatusColorsString();
 772
 773
 774         //copied from DescriptionListController
 775
 776         boolean ignoreDistributionStatusUndefined = true;  //workaround until #9500 is fully implemented
 777         distributionConfig.setIgnoreDistributionStatusUndefined(ignoreDistributionStatusUndefined);
 778         boolean fallbackAsParent = true;  //may become a service parameter in future
 779
 780         DistributionInfoDto dto;
 781
 782         //hiddenArea markers include markers for fully hidden areas and fallback areas. The later
 783         //are hidden markers on areas that have non-hidden subareas (#4408)
 784         Set<MarkerType> hiddenAreaMarkerTypes = distributionConfig.getHiddenAreaMarkerTypeList();
 785         if(hiddenAreaMarkerTypes != null && !hiddenAreaMarkerTypes.isEmpty()){
 786             condensedConfig.hiddenAndFallbackAreaMarkers = hiddenAreaMarkerTypes.stream().map(mt->mt.getUuid()).collect(Collectors.toSet());
 787         }
 788
 789         List<String> initStrategy = null;
 790
 791         Map<PresenceAbsenceTerm, Color> distributionStatusColors;
 792         try {
 793             distributionStatusColors = DistributionServiceUtilities.buildStatusColorMap(
 794                     statusColorsString, repository.getTermService(), repository.getVocabularyService());
 795         } catch (JsonProcessingException e) {
 796             logger.error("JsonProcessingException when reading distribution status colors");
 797             //TODO is null allowed?
 798             distributionStatusColors = null;
 799         }
 800
 801         dto = distributionService.composeDistributionInfoFor(distributionConfig, taxon.getUuid(),
 802                 fallbackAsParent,
 803                 distributionStatusColors, LocaleContext.getLanguages(),
 804                 initStrategy);
 805
 806         if (distributionConfig.isUseTreeDto() && dto.getTree() != null) {
 807             DistributionTreeDto tree = (DistributionTreeDto)dto.getTree();
 808             TreeNode<Set<DistributionDto>, NamedAreaDto> root = tree.getRootElement();
 809             //fill uuid->distribution map
 810             Map<UUID,Distribution> distributionMap = new HashMap<>();
 811             distributions.stream().forEach(d->distributionMap.put(d.getUuid(), d));
 812             handleDistributionDtoNode(distributionMap, root);
 813         }
 814
 815         featureDto.addFact(dto);
 816     }
 817
 818     private void handleDistributionDtoNode(Map<UUID, Distribution> map,
 819             TreeNode<Set<DistributionDto>, NamedAreaDto> root) {
 820        if (root.getData() != null) {
 821            root.getData().stream().forEach(d->{
 822                Distribution distr  = map.get(d.getUuid());
 823                loadBaseData(distr, d);
 824                d.setTimeperiod(distr.getTimeperiod() == null ? null : distr.getTimeperiod().toString());
 825
 826            });
 827        }
 828        //handle children
 829        if (root.getChildren() != null) {
 830            root.getChildren().stream().forEach(c->handleDistributionDtoNode(map, c));
 831        }
 832     }
 833
 834     private FactDtoBase handleFact(FeatureDto featureDto, DescriptionElementBase fact) {
 835         //TODO locale
 836         Language localeLang = null;
 837
 838         FactDtoBase result;
 839         if (fact.isInstanceOf(TextData.class)) {
 840             TextData td = CdmBase.deproxy(fact, TextData.class);
 841             LanguageString ls = td.getPreferredLanguageString(localeLang);
 842             String text = ls == null ? "" : CdmUtils.Nz(ls.getText());
 843
 844             FactDto factDto = new FactDto();
 845             featureDto.addFact(factDto);
 846             //TODO do we really need type information for textdata here?
 847             TypedLabel typedLabel = new TypedLabel(text);
 848             typedLabel.setClassAndId(td);
 849             factDto.getTypedLabel().add(typedLabel);
 850             loadBaseData(td, factDto);
 851             //TODO
 852             result = factDto;
 853         }else if (fact.isInstanceOf(CommonTaxonName.class)) {
 854             CommonTaxonName ctn = CdmBase.deproxy(fact, CommonTaxonName.class);
 855             CommonNameDto dto = new CommonNameDto();
 856             featureDto.addFact(dto);
 857
 858             Language lang = ctn.getLanguage();
 859             if (lang != null) {
 860                 String langLabel = getTermLabel(lang, localeLang);
 861                 dto.setLanguage(langLabel);
 862                 dto.setLanguageUuid(lang.getUuid());
 863             }else {
 864                 //TODO
 865                 dto.setLanguage("-");
 866             }
 867             //area
 868             NamedArea area = ctn.getArea();
 869             if (area != null) {
 870                 String areaLabel = getTermLabel(area, localeLang);
 871                 dto.setArea(areaLabel);
 872                 dto.setAreaUUID(area.getUuid());
 873             }
 874             dto.setName(ctn.getName());
 875             loadBaseData(ctn, dto);
 876             //TODO sort all common names
 877             result = dto;
 878         } else if (fact.isInstanceOf(IndividualsAssociation.class)) {
 879             IndividualsAssociation ia = CdmBase.deproxy(fact, IndividualsAssociation.class);
 880             IndividualsAssociationDto dto = new IndividualsAssociationDto ();
 881
 882             LanguageString description = MultilanguageTextHelper.getPreferredLanguageString(ia.getDescription(), Arrays.asList(localeLang));
 883             if (description != null) {
 884                 dto.setDescritpion(description.getText());
 885             }
 886             SpecimenOrObservationBase<?> specimen = ia.getAssociatedSpecimenOrObservation();
 887             if (specimen != null) {
 888                 //TODO what to use here??
 889                 dto.setOccurrence(specimen.getTitleCache());
 890                 dto.setOccurrenceUuid(specimen.getUuid());
 891             }
 892
 893             featureDto.addFact(dto);
 894             loadBaseData(ia, dto);
 895             result = dto;
 896         } else if (fact.isInstanceOf(TaxonInteraction.class)) {
 897             TaxonInteraction ti = CdmBase.deproxy(fact, TaxonInteraction.class);
 898             TaxonInteractionDto dto = new TaxonInteractionDto ();
 899
 900             LanguageString description = MultilanguageTextHelper.getPreferredLanguageString(
 901                     ti.getDescription(), Arrays.asList(localeLang));
 902             if (description != null) {
 903                 dto.setDescritpion(description.getText());
 904             }
 905             Taxon taxon = ti.getTaxon2();
 906             if (taxon != null) {
 907                 //TODO what to use here??
 908                 dto.setTaxon(taxon.cacheStrategy().getTaggedTitle(taxon));
 909                 dto.setTaxonUuid(taxon.getUuid());
 910             }
 911             featureDto.addFact(dto);
 912             loadBaseData(ti, dto);
 913             result = dto;
 914         }else if (fact.isInstanceOf(CategoricalData.class)) {
 915             CategoricalData cd = CdmBase.deproxy(fact, CategoricalData.class);
 916             FactDto factDto = new FactDto();
 917             featureDto.addFact(factDto);
 918             //TODO do we really need type information for textdata here?
 919             String label = CategoricalDataFormatter.NewInstance(null).format(cd, localeLang);
 920             TypedLabel typedLabel = new TypedLabel(label);
 921             typedLabel.setClassAndId(cd);
 922             factDto.getTypedLabel().add(typedLabel);
 923             //TODO
 924             loadBaseData(cd, factDto);
 925             result = factDto;
 926         }else if (fact.isInstanceOf(QuantitativeData.class)) {
 927             QuantitativeData qd = CdmBase.deproxy(fact, QuantitativeData.class);
 928             FactDto factDto = new FactDto();
 929             featureDto.addFact(factDto);
 930             //TODO do we really need type information for textdata here?
 931             String label = QuantitativeDataFormatter.NewInstance(null).format(qd, localeLang);
 932             TypedLabel typedLabel = new TypedLabel(label);
 933             typedLabel.setClassAndId(qd);
 934             factDto.getTypedLabel().add(typedLabel);
 935             //TODO
 936             loadBaseData(qd, factDto);
 937             result = factDto;
 938         }else if (fact.isInstanceOf(TemporalData.class)) {
 939             TemporalData td = CdmBase.deproxy(fact, TemporalData.class);
 940             FactDto factDto = new FactDto();
 941             featureDto.addFact(factDto);
 942             //TODO do we really need type information for textdata here?
 943             String label = td.toString();
 944             TypedLabel typedLabel = new TypedLabel(label);
 945             typedLabel.setClassAndId(td);
 946             factDto.getTypedLabel().add(typedLabel);
 947             //TODO
 948             loadBaseData(td, factDto);
 949             result = factDto;
 950         }else {
 951 //            TODO
 952             logger.warn("DescriptionElement type not yet handled: " + fact.getClass().getSimpleName());
 953             return null;
 954         }
 955         result.setTimeperiod(fact.getTimeperiod() == null ? null : fact.getTimeperiod().toString());
 956         return result;
 957     }
 958
 959     private String getTermLabel(TermBase term, Language localeLang) {
 960         if (term == null) {
 961             return null;
 962         }
 963         Representation rep = term.getPreferredRepresentation(localeLang);
 964         String label = rep == null ? null : rep.getLabel();
 965         label = label == null ? term.getLabel() : label;
 966         return label;
 967     }
 968
 969     /**
 970      * Compares an existing last date and the last date of an entity
 971      * and returns the resulting last date.
 972      */
 973     private LocalDateTime getLastUpdated(LocalDateTime existingLastDate, VersionableEntity dateToAddEntity) {
 974
 975         DateTime dateToAdd = dateToAddEntity.getUpdated() != null ? dateToAddEntity.getUpdated() : dateToAddEntity.getCreated();
 976
 977         LocalDateTime javaLocalDateTimeOfEntity = dateToAdd == null ? null:
 978                 LocalDateTime.of(dateToAdd.getYear(), dateToAdd.getMonthOfYear(),
 979                         dateToAdd.getDayOfMonth(), dateToAdd.getHourOfDay(),
 980                         dateToAdd.getMinuteOfHour(), dateToAdd.getSecondOfMinute());
 981
 982        if (existingLastDate == null) {
 983            return javaLocalDateTimeOfEntity;
 984        }else if (javaLocalDateTimeOfEntity == null || javaLocalDateTimeOfEntity.compareTo(existingLastDate) < 0)  {
 985            return existingLastDate;
 986        }else {
 987            return javaLocalDateTimeOfEntity;
 988        }
 989     }
 990
 991     private void loadBaseData(CdmBase cdmBase, CdmBaseDto dto) {
 992         dto.setId(cdmBase.getId());
 993         dto.setUuid(cdmBase.getUuid());
 994
 995         loadAnnotatable(cdmBase, dto);
 996         loadSources(cdmBase, dto);
 997         //loadIdentifiable(cdmBase, dto);
 998     }
 999
1000     private void loadSources(CdmBase cdmBase, CdmBaseDto dto) {
1001         if (dto instanceof SingleSourcedDto && cdmBase.isInstanceOf(SingleSourcedEntityBase.class)) {
1002             //TODO other sourced
1003             SingleSourcedEntityBase sourced = CdmBase.deproxy(cdmBase, SingleSourcedEntityBase.class);
1004             SingleSourcedDto sourcedDto = (SingleSourcedDto)dto;
1005             NamedSource source = sourced.getSource();
1006             SourceDto sourceDto = new SourceDto();
1007             loadSource(source, sourceDto);
1008             sourcedDto.setSource(sourceDto);
1009         } else if (dto instanceof SourcedDto && cdmBase instanceof ISourceable) {
1010             @SuppressWarnings("unchecked")
1011             ISourceable<OriginalSourceBase> sourced = (ISourceable<OriginalSourceBase>)cdmBase;
1012             SourcedDto sourcedDto = (SourcedDto)dto;
1013             for (OriginalSourceBase source : sourced.getSources()) {
1014                 SourceDto sourceDto = new SourceDto();
1015                 loadSource(source, sourceDto);
1016                 sourcedDto.addSource(sourceDto);
1017             }
1018         }
1019     }
1020
1021     private void loadSource(OriginalSourceBase source, SourceDto sourceDto) {
1022
1023         source = CdmBase.deproxy(source);
1024         //base data
1025         loadBaseData(source, sourceDto);
1026
1027         ICdmBase linkedObject = source.getCitation();
1028         if (linkedObject == null) {
1029             //cdmsource
1030             linkedObject = source.getCdmSource();
1031         }
1032
1033         //citation uuid & doi
1034         if (source.getCitation()!= null) {
1035             sourceDto.setDoi(source.getCitation().getDoiString());
1036         }
1037
1038         //label
1039         //TODO this creates only the short form and probably also does not use
1040         //     specimen or cdmSource if necessary
1041         String label = OriginalSourceFormatter.INSTANCE.format(source);
1042         TypedLabel typedLabel = new TypedLabel(source, label);
1043         sourceDto.addLabel(typedLabel);
1044         sourceDto.setType(source.getType() != null ? source.getType().toString() : null);
1045
1046         if (source.isInstanceOf(NamedSourceBase.class)) {
1047             NamedSourceBase ns = CdmBase.deproxy(source, NamedSourceBase.class);
1048
1049             //nameUsedInSource
1050             TaxonName name =  ns.getNameUsedInSource();
1051             if (name != null) {
1052                 List<TaggedText> taggedName = name.cacheStrategy().getTaggedTitle(name);
1053                 //TODO nom status?
1054                 sourceDto.setNameInSource(taggedName);
1055                 sourceDto.setNameInSourceUuid(name.getUuid());
1056             }
1057
1058             //specimen uuid
1059             if (source.isInstanceOf(DescriptionElementSource.class)) {
1060                 DescriptionElementSource des = CdmBase.deproxy(source, DescriptionElementSource.class);
1061                 if (linkedObject == null) {
1062                     linkedObject = des.getSpecimen();
1063                 }
1064             }
1065         }
1066
1067         sourceDto.setLinkedUuid(getUuid(linkedObject));
1068         String linkedObjectStr = linkedObject == null ? null : CdmBase.deproxy(linkedObject).getClass().getSimpleName();
1069         sourceDto.setLinkedClass(linkedObjectStr);
1070     }
1071
1072     private UUID getUuid(ICdmBase cdmBase) {
1073         return cdmBase == null ? null : cdmBase.getUuid();
1074     }
1075
1076     private void loadAnnotatable(CdmBase cdmBase, CdmBaseDto dto) {
1077         if (dto instanceof AnnotatableDto && cdmBase.isInstanceOf(AnnotatableEntity.class)) {
1078             AnnotatableEntity annotatable = CdmBase.deproxy(cdmBase, AnnotatableEntity.class);
1079             AnnotatableDto annotatableDto = (AnnotatableDto)dto;
1080             //annotation
1081             for (Annotation annotation : annotatable.getAnnotations()) {
1082                 if (annotation.getAnnotationType() != null
1083                         //TODO annotation type filter
1084                         && annotation.getAnnotationType().getUuid().equals(AnnotationType.uuidEditorial)
1085                         && StringUtils.isNotBlank(annotation.getText())) {
1086
1087                     AnnotationDto annotationDto = new AnnotationDto();
1088                     annotatableDto.addAnnotation(annotationDto);
1089                     //TODO id needed? but need to adapt dto and container then
1090                     loadBaseData(annotation, annotationDto);
1091                     annotationDto.setText(annotation.getText());
1092                     UUID uuidAnnotationType = annotation.getAnnotationType() == null ? null :annotation.getAnnotationType().getUuid();
1093                     annotationDto.setTypeUuid(uuidAnnotationType);
1094                     //language etc. currently not yet used
1095                 }
1096             }
1097
1098             //marker
1099             for (Marker marker : annotatable.getMarkers()) {
1100                 if (marker.getMarkerType() != null
1101                         //TODO markertype filter
1102 //                        && marker.getMarkerType().getUuid().equals(AnnotationType.uuidEditorial)
1103                            ){
1104
1105                     MarkerDto markerDto = new MarkerDto();
1106                     annotatableDto.addMarker(markerDto);
1107                     //TODO id needed? but need to adapt dto and container then
1108                     loadBaseData(marker, markerDto);
1109                     if (marker.getMarkerType() != null) {
1110                         markerDto.setTypeUuid(marker.getMarkerType().getUuid());
1111                         //TODO locale
1112                         markerDto.setType(marker.getMarkerType().getTitleCache());
1113                     }
1114                     markerDto.setValue(marker.getValue());
1115                 }
1116             }
1117         }
1118     }
1119 }