cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/portal/PortalDtoLoader.java

   1 /**
   2 * Copyright (C) 2023 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.api.service.portal;
  10
  11 import java.awt.Color;
  12 import java.time.LocalDateTime;
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.Collections;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22 import java.util.stream.Collectors;
  23
  24 import org.apache.commons.lang3.StringUtils;
  25 import org.apache.logging.log4j.LogManager;
  26 import org.apache.logging.log4j.Logger;
  27 import org.joda.time.DateTime;
  28
  29 import com.fasterxml.jackson.core.JsonProcessingException;
  30
  31 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  32 import eu.etaxonomy.cdm.api.dto.portal.AnnotatableDto;
  33 import eu.etaxonomy.cdm.api.dto.portal.AnnotationDto;
  34 import eu.etaxonomy.cdm.api.dto.portal.CdmBaseDto;
  35 import eu.etaxonomy.cdm.api.dto.portal.CommonNameDto;
  36 import eu.etaxonomy.cdm.api.dto.portal.ContainerDto;
  37 import eu.etaxonomy.cdm.api.dto.portal.DistributionInfoDto;
  38 import eu.etaxonomy.cdm.api.dto.portal.FactDto;
  39 import eu.etaxonomy.cdm.api.dto.portal.FactDtoBase;
  40 import eu.etaxonomy.cdm.api.dto.portal.FeatureDto;
  41 import eu.etaxonomy.cdm.api.dto.portal.IFactDto;
  42 import eu.etaxonomy.cdm.api.dto.portal.IndividualsAssociationDto;
  43 import eu.etaxonomy.cdm.api.dto.portal.MarkerDto;
  44 import eu.etaxonomy.cdm.api.dto.portal.MessagesDto;
  45 import eu.etaxonomy.cdm.api.dto.portal.SingleSourcedDto;
  46 import eu.etaxonomy.cdm.api.dto.portal.SourceDto;
  47 import eu.etaxonomy.cdm.api.dto.portal.SourcedDto;
  48 import eu.etaxonomy.cdm.api.dto.portal.TaxonBaseDto;
  49 import eu.etaxonomy.cdm.api.dto.portal.TaxonBaseDto.TaxonNameDto;
  50 import eu.etaxonomy.cdm.api.dto.portal.TaxonInteractionDto;
  51 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto;
  52 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.ConceptRelationDTO;
  53 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.HomotypicGroupDTO;
  54 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.KeyDTO;
  55 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.MediaDTO;
  56 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.MediaRepresentationDTO;
  57 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.NameRelationDTO;
  58 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.SpecimenDTO;
  59 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.TaxonNodeAgentsRelDTO;
  60 import eu.etaxonomy.cdm.api.dto.portal.TaxonPageDto.TaxonNodeDTO;
  61 import eu.etaxonomy.cdm.api.dto.portal.config.DistributionInfoConfiguration;
  62 import eu.etaxonomy.cdm.api.dto.portal.config.TaxonPageDtoConfiguration;
  63 import eu.etaxonomy.cdm.api.service.geo.DistributionServiceUtilities;
  64 import eu.etaxonomy.cdm.api.service.geo.IDistributionService;
  65 import eu.etaxonomy.cdm.api.service.l10n.LocaleContext;
  66 import eu.etaxonomy.cdm.api.service.name.TypeDesignationSetContainer;
  67 import eu.etaxonomy.cdm.api.service.name.TypeDesignationSetFormatter;
  68 import eu.etaxonomy.cdm.api.service.pager.Pager;
  69 import eu.etaxonomy.cdm.common.CdmUtils;
  70 import eu.etaxonomy.cdm.common.TreeNode;
  71 import eu.etaxonomy.cdm.compare.taxon.TaxonComparator;
  72 import eu.etaxonomy.cdm.format.common.TypedLabel;
  73 import eu.etaxonomy.cdm.format.description.CategoricalDataFormatter;
  74 import eu.etaxonomy.cdm.format.description.QuantitativeDataFormatter;
  75 import eu.etaxonomy.cdm.format.description.distribution.CondensedDistributionConfiguration;
  76 import eu.etaxonomy.cdm.format.reference.OriginalSourceFormatter;
  77 import eu.etaxonomy.cdm.format.taxon.TaxonRelationshipFormatter;
  78 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  79 import eu.etaxonomy.cdm.model.common.Annotation;
  80 import eu.etaxonomy.cdm.model.common.AnnotationType;
  81 import eu.etaxonomy.cdm.model.common.CdmBase;
  82 import eu.etaxonomy.cdm.model.common.ICdmBase;
  83 import eu.etaxonomy.cdm.model.common.IPublishable;
  84 import eu.etaxonomy.cdm.model.common.Language;
  85 import eu.etaxonomy.cdm.model.common.LanguageString;
  86 import eu.etaxonomy.cdm.model.common.Marker;
  87 import eu.etaxonomy.cdm.model.common.MarkerType;
  88 import eu.etaxonomy.cdm.model.common.MultilanguageTextHelper;
  89 import eu.etaxonomy.cdm.model.common.SingleSourcedEntityBase;
  90 import eu.etaxonomy.cdm.model.common.VersionableEntity;
  91 import eu.etaxonomy.cdm.model.description.CategoricalData;
  92 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  93 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  94 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  95 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  96 import eu.etaxonomy.cdm.model.description.Distribution;
  97 import eu.etaxonomy.cdm.model.description.Feature;
  98 import eu.etaxonomy.cdm.model.description.IDescribable;
  99 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
 100 import eu.etaxonomy.cdm.model.description.PolytomousKey;
 101 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
 102 import eu.etaxonomy.cdm.model.description.QuantitativeData;
 103 import eu.etaxonomy.cdm.model.description.TaxonDescription;
 104 import eu.etaxonomy.cdm.model.description.TaxonInteraction;
 105 import eu.etaxonomy.cdm.model.description.TemporalData;
 106 import eu.etaxonomy.cdm.model.description.TextData;
 107 import eu.etaxonomy.cdm.model.location.NamedArea;
 108 import eu.etaxonomy.cdm.model.media.ExternalLink;
 109 import eu.etaxonomy.cdm.model.media.ImageFile;
 110 import eu.etaxonomy.cdm.model.media.Media;
 111 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
 112 import eu.etaxonomy.cdm.model.media.MediaRepresentationPart;
 113 import eu.etaxonomy.cdm.model.name.HomotypicalGroup;
 114 import eu.etaxonomy.cdm.model.name.NameRelationship;
 115 import eu.etaxonomy.cdm.model.name.NameRelationshipType;
 116 import eu.etaxonomy.cdm.model.name.NomenclaturalSource;
 117 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
 118 import eu.etaxonomy.cdm.model.name.TaxonName;
 119 import eu.etaxonomy.cdm.model.name.TypeDesignationBase;
 120 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
 121 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
 122 import eu.etaxonomy.cdm.model.reference.ISourceable;
 123 import eu.etaxonomy.cdm.model.reference.NamedSource;
 124 import eu.etaxonomy.cdm.model.reference.NamedSourceBase;
 125 import eu.etaxonomy.cdm.model.reference.OriginalSourceBase;
 126 import eu.etaxonomy.cdm.model.reference.Reference;
 127 import eu.etaxonomy.cdm.model.taxon.Classification;
 128 import eu.etaxonomy.cdm.model.taxon.Synonym;
 129 import eu.etaxonomy.cdm.model.taxon.Taxon;
 130 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
 131 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
 132 import eu.etaxonomy.cdm.model.taxon.TaxonNodeAgentRelation;
 133 import eu.etaxonomy.cdm.model.taxon.TaxonNodeStatus;
 134 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
 135 import eu.etaxonomy.cdm.model.term.Representation;
 136 import eu.etaxonomy.cdm.model.term.TermBase;
 137 import eu.etaxonomy.cdm.model.term.TermNode;
 138 import eu.etaxonomy.cdm.model.term.TermTree;
 139 import eu.etaxonomy.cdm.strategy.cache.TaggedCacheHelper;
 140 import eu.etaxonomy.cdm.strategy.cache.TaggedText;
 141 import eu.etaxonomy.cdm.strategy.cache.name.INameCacheStrategy;
 142 import eu.etaxonomy.cdm.strategy.cache.taxon.TaxonBaseDefaultCacheStrategy;
 143
 144 /**
 145  * Loads the portal dto from a taxon instance.
 146  * Maybe later also supports loading from persistence.
 147  *
 148  * @author a.mueller
 149  * @date 09.01.2023
 150  */
 151 public class PortalDtoLoader {
 152
 153     @SuppressWarnings("unused")
 154     private static final Logger logger = LogManager.getLogger();
 155
 156     private ICdmRepository repository;
 157
 158     public PortalDtoLoader(ICdmRepository repository) {
 159         this.repository = repository;
 160     }
 161
 162     public TaxonPageDto load(Taxon taxon, TaxonPageDtoConfiguration config) {
 163         TaxonPageDto result = new TaxonPageDto();
 164
 165         loadAcceptedTaxon(taxon, config, result);
 166
 167         loadTaxonNodes(taxon, result, config);
 168         loadSynonyms(taxon, result, config);
 169         loadConceptRelations(taxon, result, config);
 170         loadFacts(taxon, result, config);
 171         loadMedia(taxon, result, config);
 172         loadSpecimens(taxon, result, config);
 173         loadKeys(taxon, result, config);
 174
 175         return result;
 176     }
 177
 178     private void loadAcceptedTaxon(Taxon taxon, TaxonPageDtoConfiguration config, TaxonPageDto result) {
 179         try {
 180             TaxonName name = taxon.getName();
 181
 182             //load 1:1
 183             //TODO supplementalData for name
 184             loadBaseData(taxon, result);
 185             result.setLastUpdated(getLastUpdated(null, taxon));
 186             result.setLabel(CdmUtils.Nz(taxon.getTitleCache()));
 187 //          result.setTypedTaxonLabel(getTypedTaxonLabel(taxon, config));
 188             result.setTaggedLabel(getTaggedTaxon(taxon, config));
 189             if (name != null) {
 190                 handleName(config, result, name, result);
 191             }
 192             if (taxon.getSec() != null) {
 193                 result.setSecTitleCache(taxon.getSec().getTitleCache());
 194             }
 195         } catch (Exception e) {
 196             //e.printStackTrace();
 197             result.addMessage(MessagesDto.NewErrorInstance("Error when loading accepted name data.", e));
 198         }
 199     }
 200
 201     private void handleName(TaxonPageDtoConfiguration config, TaxonBaseDto taxonDto, TaxonName name, TaxonPageDto pageDto) {
 202         TaxonNameDto nameDto = taxonDto.new TaxonNameDto();
 203         loadBaseData(name, nameDto);
 204
 205         INameCacheStrategy formatter = name.cacheStrategy();
 206         formatter.setEtAlPosition(config.getEtAlPosition());
 207
 208         taxonDto.setName(nameDto);
 209         taxonDto.setNameLabel(formatter.getTitleCache(name));
 210         handleRelatedNames(name, taxonDto, config);
 211         loadProtologues(name, taxonDto);
 212         taxonDto.setNameUuid(name.getUuid());
 213         taxonDto.setNameType(name.getNameType().toString());
 214         loadNameFacts(name, taxonDto, config, pageDto);
 215         nameDto.setTaggedName(formatter.getTaggedFullTitle(name));
 216     }
 217
 218     private List<TaggedText> getTaggedTaxon(TaxonBase<?> taxon, TaxonPageDtoConfiguration config) {
 219 //        List<TypedLabel> result = new ArrayList<>();
 220         TaxonBaseDefaultCacheStrategy<TaxonBase<?>> formatter = new TaxonBaseDefaultCacheStrategy<>();
 221         List<TaggedText> tags = formatter.getTaggedTitle(taxon);
 222         return tags;
 223     }
 224
 225     private void loadKeys(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 226         try {
 227             ContainerDto<KeyDTO> container = new ContainerDto<>();
 228
 229             //TODO other key types, but type must not be null, otherwise NPE
 230             Pager<PolytomousKey> keys = repository.getIdentificationKeyService().findKeysConvering(taxon, PolytomousKey.class, null, null, null);
 231             for (PolytomousKey key : keys.getRecords()) {
 232                 KeyDTO dto = new KeyDTO();
 233                 loadBaseData(key, dto);
 234                 dto.setLabel(key.getTitleCache());
 235                 dto.setKeyClass(key.getClass().getSimpleName());
 236                 container.addItem(dto);
 237             }
 238             if (container.getCount() > 0) {
 239                 result.setKeys(container);
 240             }
 241         } catch (Exception e) {
 242             //e.printStackTrace();
 243             result.addMessage(MessagesDto.NewErrorInstance("Error when loading identification key data.", e));
 244         }
 245     }
 246
 247     private void loadSpecimens(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 248         //TODO load specimen from multiple places
 249
 250         try {
 251             ContainerDto<SpecimenDTO> container = new ContainerDto<>();
 252
 253             List<SpecimenOrObservationBase<?>> specimens = new ArrayList<>();
 254             for (TaxonDescription taxonDescription : taxon.getDescriptions()) {
 255                 if (taxonDescription.isImageGallery()) {
 256                     continue;
 257                 }
 258                 for (DescriptionElementBase el : taxonDescription.getElements()) {
 259                     if (el.isInstanceOf(IndividualsAssociation.class)) {
 260                         IndividualsAssociation indAss = CdmBase.deproxy(el, IndividualsAssociation.class);
 261                         SpecimenOrObservationBase<?> specimen = indAss.getAssociatedSpecimenOrObservation();
 262                         specimens.add(specimen);
 263                     }
 264                 }
 265             }
 266             List<SpecimenOrObservationBase<?>> typeSpecimens = loadTypeSpecimen(taxon.getName(), config);
 267             specimens.addAll(typeSpecimens);
 268             for (TaxonName syn : taxon.getSynonymNames()) {
 269                 typeSpecimens = loadTypeSpecimen(syn, config);
 270                 specimens.addAll(typeSpecimens);
 271             }
 272
 273             //TODO maybe still need to check full derivate path #4484, #8424, #9559
 274             for (SpecimenOrObservationBase<?> specimen : filterPublished(specimens)) {
 275                 SpecimenDTO dto = new SpecimenDTO();
 276                 loadBaseData(specimen, dto);
 277                 dto.setLabel(specimen.getTitleCache());
 278                 container.addItem(dto);
 279             }
 280             if (container.getCount() > 0 ) {
 281                 result.setSpecimens(container);
 282             }
 283         } catch (Exception e) {
 284             //e.printStackTrace();
 285             result.addMessage(MessagesDto.NewErrorInstance("Error when loading specimen data.", e));
 286         }
 287     }
 288
 289     private List<SpecimenOrObservationBase<?>> loadTypeSpecimen(TaxonName name, TaxonPageDtoConfiguration config) {
 290         List<SpecimenOrObservationBase<?>> result = new ArrayList<>();
 291         for (SpecimenTypeDesignation desig: name.getSpecimenTypeDesignations()){
 292             DerivedUnit specimen = desig.getTypeSpecimen();
 293             if (specimen != null) {
 294                 result.add(specimen);
 295             }
 296         }
 297         return result;
 298     }
 299
 300     private void loadMedia(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 301
 302         try {
 303             ContainerDto<MediaDTO> container = new ContainerDto<TaxonPageDto.MediaDTO>();
 304
 305             List<Media> medias = new ArrayList<>();
 306             for (TaxonDescription taxonDescription : taxon.getDescriptions()) {
 307                 if (!taxonDescription.isImageGallery()) {
 308                     continue;
 309                 }
 310
 311                 List<Media> newMedia = taxonDescription.getElements().stream()
 312                     .filter(el->el.isInstanceOf(TextData.class))
 313                     .map(el->CdmBase.deproxy(el, TextData.class))
 314                     .filter(td->true)
 315                     .flatMap(td->td.getMedia().stream())
 316                     .collect(Collectors.toList())
 317                     ;
 318                 medias.addAll(newMedia);
 319             }
 320             //TODO collect media from elsewhere
 321             for (Media media : medias) {
 322                 MediaDTO dto = new TaxonPageDto.MediaDTO();
 323                 loadBaseData(media, dto);
 324                 dto.setLabel(media.getTitleCache());
 325                 ContainerDto<MediaRepresentationDTO> representations = new ContainerDto<>();
 326                 for (MediaRepresentation rep : media.getRepresentations()) {
 327                     MediaRepresentationDTO repDto = new MediaRepresentationDTO();
 328                     loadBaseData(rep, dto);
 329                     repDto.setMimeType(rep.getMimeType());
 330                     repDto.setSuffix(rep.getSuffix());
 331                     if (!rep.getParts().isEmpty()) {
 332                         //TODO handle message if n(parts) > 1
 333                         MediaRepresentationPart part = rep.getParts().get(0);
 334                         repDto.setUri(part.getUri());
 335                         repDto.setClazz(part.getClass());
 336                         repDto.setSize(part.getSize());
 337                         if (part.isInstanceOf(ImageFile.class)) {
 338                             ImageFile image = CdmBase.deproxy(part, ImageFile.class);
 339                             repDto.setHeight(image.getHeight());
 340                             repDto.setWidth(image.getWidth());
 341                         }
 342                         //TODO AudioFile etc.
 343                     }
 344                     representations.addItem(repDto);
 345                 }
 346                 if (representations.getCount() > 0) {
 347                     dto.setRepresentations(representations);
 348                 }
 349                 //TODO load representation data
 350                 container.addItem(dto);
 351             }
 352
 353             if (container.getCount() > 0) {
 354                 result.setMedia(container);
 355             }
 356         } catch (Exception e) {
 357             //e.printStackTrace();
 358             result.addMessage(MessagesDto.NewErrorInstance("Error when loading media data.", e));
 359         }
 360     }
 361
 362     private void loadTaxonNodes(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 363         try {
 364             ContainerDto<TaxonNodeDTO> container = new ContainerDto<TaxonPageDto.TaxonNodeDTO>();
 365             for (TaxonNode node : taxon.getTaxonNodes()) {
 366                 TaxonNodeDTO dto = new TaxonNodeDTO();
 367                 loadBaseData(node, dto);
 368                 //classification
 369                 Classification classification = node.getClassification();
 370                 if (classification != null) {
 371                     dto.setClassificationUuid(node.getClassification().getUuid());
 372                     dto.setClassificationLabel(classification.getName().getText());
 373                 }
 374                 //TODO lang/locale
 375                 Language language = Language.DEFAULT();
 376
 377                 //status
 378                 TaxonNodeStatus status = node.getStatus();
 379                 if (status != null) {
 380                     dto.setStatus(status.getLabel(language));
 381                 }
 382                 //statusNote
 383                 Map<Language, LanguageString> statusNote = node.getStatusNote();
 384                 if (statusNote != null) {
 385                     //TODO handle fallback lang
 386                     LanguageString statusNoteStr = statusNote.get(language);
 387                     if (statusNoteStr == null && statusNote.size() > 0) {
 388                         statusNoteStr = statusNote.entrySet().iterator().next().getValue();
 389                     }
 390                     if (statusNoteStr != null) {
 391                         dto.setStatusNote(statusNoteStr.getText());
 392                     }
 393                 }
 394                 //agent relations
 395                 Set<TaxonNodeAgentRelation> agents = node.getAgentRelations();
 396                 if (!agents.isEmpty()) {
 397                     for (TaxonNodeAgentRelation rel : agents) {
 398                         TaxonNodeAgentsRelDTO agentDto = new TaxonNodeAgentsRelDTO();
 399                         loadBaseData(rel, agentDto);
 400
 401                         //TODO laod
 402                         if (rel.getAgent() != null) {
 403                             agentDto.setAgent(rel.getAgent().getFullTitle());
 404                             agentDto.setAgentUuid(rel.getAgent().getUuid());
 405                             //TODO compute preferred external link
 406                             agentDto.setAgentLink(null);
 407                         }
 408                         if (rel.getType() != null) {
 409                             agentDto.setType(rel.getType().getTitleCache());
 410                             agentDto.setTypeUuid(rel.getType().getUuid());
 411                         }
 412                         dto.addAgent(agentDto);
 413                     }
 414                 }
 415                 container.addItem(dto);
 416             }
 417             if (container.getCount() > 0) {
 418                 result.setTaxonNodes(container);
 419             }
 420         } catch (Exception e) {
 421 //            e.printStackTrace();
 422             result.addMessage(MessagesDto.NewErrorInstance("Error when loading taxon node data.", e));
 423         }
 424     }
 425
 426     private void loadSynonyms(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 427
 428         try {
 429             //        List<HomotypicalGroup> homotypicGroups = taxon.getHomotypicSynonymyGroups();
 430
 431             TaxonComparator comparator = new TaxonComparator();
 432
 433             TaxonName name = taxon.getName();
 434
 435             //TODO depending on config add/remove accepted name
 436
 437             //homotypic synonyms
 438             List<Synonym> homotypicSynonmys = filterPublished(taxon.getHomotypicSynonymsByHomotypicGroup(comparator));
 439
 440             TaxonPageDto.HomotypicGroupDTO homotypicGroupDto = new TaxonPageDto.HomotypicGroupDTO();
 441             if (homotypicSynonmys != null && !homotypicSynonmys.isEmpty()) {
 442                 loadBaseData(name.getHomotypicalGroup(), homotypicGroupDto);
 443
 444                 for (Synonym syn : homotypicSynonmys) {
 445                     loadSynonymsInGroup(homotypicGroupDto, syn, config, result);
 446                 }
 447             }
 448             if (name != null) {
 449                 handleTypification(name.getHomotypicalGroup(), homotypicGroupDto, result, config);
 450             }
 451             result.setHomotypicSynonyms(homotypicGroupDto);
 452
 453             //heterotypic synonyms
 454             List<HomotypicalGroup> heteroGroups = taxon.getHeterotypicSynonymyGroups();
 455             if (heteroGroups.isEmpty()) {
 456                 return;
 457             }
 458             ContainerDto<HomotypicGroupDTO> heteroContainer = new ContainerDto<>();
 459             result.setHeterotypicSynonymGroups(heteroContainer);
 460
 461             for (HomotypicalGroup hg : heteroGroups) {
 462                 TaxonPageDto.HomotypicGroupDTO hgDto = new TaxonPageDto.HomotypicGroupDTO();
 463                 loadBaseData(taxon.getName().getHomotypicalGroup(), hgDto);
 464                 heteroContainer.addItem(hgDto);
 465
 466                 List<Synonym> heteroSyns = filterPublished(taxon.getSynonymsInGroup(hg, comparator));
 467                 for (Synonym syn : heteroSyns) {
 468                     loadSynonymsInGroup(hgDto, syn, config, result);
 469                 }
 470                 handleTypification(hg, hgDto, result, config);
 471             }
 472         } catch (Exception e) {
 473             //e.printStackTrace();
 474             result.addMessage(MessagesDto.NewErrorInstance("Error when loading synonym data.", e));
 475         }
 476     }
 477
 478     private <P extends IPublishable> List<P> filterPublished(List<P> listToPublish) {
 479         if (listToPublish == null) {
 480             return null;
 481         }else {
 482             return listToPublish.stream().filter(s->s.isPublish()).collect(Collectors.toList());
 483         }
 484     }
 485     private <P extends IPublishable> Set<P> filterPublished(Set<P> setToPublish) {
 486         if (setToPublish == null) {
 487             return null;
 488         }else {
 489             return setToPublish.stream().filter(s->s.isPublish()).collect(Collectors.toSet());
 490         }
 491     }
 492
 493     private void handleTypification(HomotypicalGroup homotypicalGroup, HomotypicGroupDTO hgDto,
 494             TaxonPageDto result, TaxonPageDtoConfiguration config) {
 495
 496         boolean withCitation = true;
 497         boolean withStartingTypeLabel = true;
 498         boolean withNameIfAvailable = false;
 499         TypeDesignationSetFormatter formatter = new TypeDesignationSetFormatter(
 500                 withCitation, withStartingTypeLabel, withNameIfAvailable);
 501         Set<TypeDesignationBase<?>> desigs = homotypicalGroup.getTypeDesignations();
 502         try {
 503             TypeDesignationSetContainer manager = TypeDesignationSetContainer.NewDefaultInstance((Set)desigs);
 504             List<TaggedText> tags = formatter.toTaggedText(manager);
 505             String label = TaggedCacheHelper.createString(tags);
 506             hgDto.setTypes(label);
 507             hgDto.setTaggedTypes(tags);
 508 //            hgDto.setTypedTypes(null);
 509
 510         } catch (Exception e) {
 511 //          e.printStackTrace();
 512             result.addMessage(MessagesDto.NewErrorInstance("Error when creating type designation information", e));
 513         }
 514     }
 515
 516     private void loadConceptRelations(Taxon taxon, TaxonPageDto result, TaxonPageDtoConfiguration config) {
 517
 518         try {
 519             //concept relations
 520             ContainerDto<ConceptRelationDTO> conceptRelContainer = new ContainerDto<>();
 521             TaxonRelationshipFormatter taxRelFormatter = TaxonRelationshipFormatter.INSTANCE();
 522
 523             //... MAN
 524             Set<TaxonRelationship> misappliedRels = taxon.getMisappliedNameRelations();
 525             for (TaxonRelationship rel : misappliedRels) {
 526                 boolean inverse = true;
 527                 boolean withoutName = false;
 528                 loadConceptRelation(taxRelFormatter, rel, conceptRelContainer, inverse, withoutName);
 529             }
 530
 531             //... pro parte Synonyms
 532             Set<TaxonRelationship> proParteRels = taxon.getProParteAndPartialSynonymRelations();
 533             for (TaxonRelationship rel : proParteRels) {
 534                 boolean inverse = true;
 535                 boolean withoutName = false;
 536                 loadConceptRelation(taxRelFormatter, rel, conceptRelContainer, inverse, withoutName);
 537             }
 538
 539             //TODO MAN and pp from this taxon
 540
 541             //... to-relations
 542             Set<TaxonRelationship> toRels = taxon.getRelationsToThisTaxon();
 543             toRels.removeAll(misappliedRels);
 544             toRels.removeAll(proParteRels);
 545             for (TaxonRelationship rel : toRels) {
 546                 boolean inverse = true;
 547                 boolean withoutName = false;
 548                 loadConceptRelation(taxRelFormatter, rel, conceptRelContainer, inverse, withoutName);
 549             }
 550
 551             //... from-relations
 552             Set<TaxonRelationship> fromRels = taxon.getRelationsFromThisTaxon();
 553             for (TaxonRelationship rel : fromRels) {
 554                 boolean inverse = false;
 555                 boolean withoutName = false;
 556                 loadConceptRelation(taxRelFormatter, rel, conceptRelContainer, inverse, withoutName);
 557             }
 558
 559             if (conceptRelContainer.getCount() > 0) {
 560                 result.setConceptRelations(conceptRelContainer);
 561             }
 562         } catch (Exception e) {
 563             //e.printStackTrace();
 564             result.addMessage(MessagesDto.NewErrorInstance("Error when loading concept relation data.", e));
 565         }
 566     }
 567
 568     private void loadConceptRelation(TaxonRelationshipFormatter taxRelFormatter, TaxonRelationship rel, ContainerDto<ConceptRelationDTO> conceptRelContainer, boolean inverse,
 569             boolean withoutName) {
 570
 571         List<Language> languages = Arrays.asList(new Language[] {Language.DEFAULT()}); // TODO config.locales;
 572         List<TaggedText> tags = taxRelFormatter.getTaggedText(rel, inverse, languages, withoutName);
 573         String relLabel = TaggedCacheHelper.createString(tags);
 574         ConceptRelationDTO dto = new TaxonPageDto.ConceptRelationDTO();
 575         loadBaseData(rel, dto);
 576         dto.setRelSource(makeSource(rel.getSource()));
 577         Taxon relTaxon = inverse ? rel.getFromTaxon() : rel.getToTaxon();
 578         dto.setRelTaxonId(relTaxon.getId());
 579         dto.setRelTaxonUuid(relTaxon.getUuid());
 580         dto.setRelTaxonLabel(relTaxon.getTitleCache());
 581         dto.setSecSource(makeSource(relTaxon.getSecSource()));
 582         dto.setLabel(relLabel);
 583         dto.setTaggedLabel(tags);
 584         dto.setNameUuid(relTaxon.getName() != null ? relTaxon.getName().getUuid() : null);
 585
 586         if (rel.getType() != null) {
 587             dto.setRelTypeUuid(rel.getType().getUuid());
 588         }
 589         for (TaxonNode node : relTaxon.getTaxonNodes()) {
 590             Classification classification = node.getClassification();
 591             if (classification != null) {
 592                 dto.addClassificationUuids(classification.getUuid());
 593             }
 594         }
 595         conceptRelContainer.addItem(dto);
 596     }
 597
 598     private void loadSynonymsInGroup(TaxonPageDto.HomotypicGroupDTO hgDto, Synonym syn,
 599             TaxonPageDtoConfiguration config, TaxonPageDto pageDto) {
 600
 601         TaxonBaseDto synDto = new TaxonBaseDto();
 602         loadBaseData(syn, synDto);
 603         synDto.setLabel(syn.getTitleCache());
 604         synDto.setTaggedLabel(getTaggedTaxon(syn, config));
 605
 606         if (syn.getName() != null) {
 607             handleName(config, synDto, syn.getName(), pageDto);
 608         }
 609
 610         //TODO
 611         hgDto.addSynonym(synDto);
 612     }
 613
 614     private void loadProtologues(TaxonName name, TaxonBaseDto taxonBaseDto) {
 615         NomenclaturalSource nomSource = name.getNomenclaturalSource();
 616         if (nomSource != null) {
 617             Set<ExternalLink> links = nomSource.getLinks();
 618             for (ExternalLink link : links) {
 619                 if (link.getUri() != null) {
 620                     taxonBaseDto.addProtologue(link.getUri());
 621                 }
 622             }
 623         }
 624     }
 625
 626     private void handleRelatedNames(TaxonName name, TaxonBaseDto taxonDto, TaxonPageDtoConfiguration config) {
 627         //exclusions TODO handle via config
 628         Set<UUID> excludedTypes = new HashSet<>();  //both directions
 629         excludedTypes.add(NameRelationshipType.uuidBasionym);
 630         excludedTypes.add(NameRelationshipType.uuidReplacedSynonym);
 631         Set<UUID> excludedFromTypes = new HashSet<>(excludedTypes);
 632         Set<UUID> excludedToTypes = new HashSet<>(excludedTypes);
 633         //TODO non-types
 634
 635         //TODO config.getLocales();
 636         Language locale = Language.DEFAULT();
 637
 638         for (NameRelationship rel : name.getRelationsFromThisName()) {
 639             TaxonName relatedName = rel.getToName();
 640             if (relatedName == null || rel.getType() == null || excludedFromTypes.contains(rel.getType().getUuid())) {
 641                 continue;
 642             }
 643             NameRelationDTO dto = new NameRelationDTO();
 644             loadBaseData(rel, dto);
 645             //name
 646             dto.setNameUuid(relatedName.getUuid());
 647             dto.setNameLabel(relatedName.getTaggedName());
 648             //type
 649             dto.setRelTypeUuid(rel.getType().getUuid());
 650             Representation rep = rel.getType().getPreferredRepresentation(locale);
 651             dto.setRelType(rep == null ? rel.getType().toString() : rep.getLabel());
 652             //inverse
 653             dto.setInverse(false);
 654             //ruleConsidered
 655             dto.setRuleConsidered(rel.getRuleConsidered());
 656             taxonDto.addRelatedName(dto);
 657         }
 658
 659         //to relations
 660         for (NameRelationship rel : name.getRelationsToThisName()) {
 661             TaxonName relatedName = rel.getFromName();
 662             if (relatedName == null || rel.getType() == null || excludedFromTypes.contains(rel.getType().getUuid())) {
 663                 continue;
 664             }
 665             NameRelationDTO dto = new NameRelationDTO();
 666             loadBaseData(rel, dto);
 667             //name
 668             dto.setNameUuid(relatedName.getUuid());
 669             dto.setNameLabel(relatedName.getTaggedName());
 670             //type
 671             dto.setRelTypeUuid(rel.getType().getUuid());
 672             Representation rep = rel.getType().getPreferredInverseRepresentation(Arrays.asList(new Language[] {locale}));
 673             dto.setRelType(rep == null ? rel.getType().toString() : rep.getLabel());
 674             //inverse
 675             dto.setInverse(true);
 676             taxonDto.addRelatedName(dto);
 677         }
 678     }
 679
 680     private void loadFacts(Taxon taxon, TaxonPageDto taxonPageDto, TaxonPageDtoConfiguration config) {
 681
 682         try {
 683             //compute the features that do exist for this taxon
 684             Map<UUID, Feature> existingFeatureUuids = getExistingFeatureUuids(taxon);
 685
 686             //filter, sort and structure according to feature tree
 687             TreeNode<Feature, UUID> filteredRootNode;
 688             if (config.getFeatureTree() != null) {
 689
 690                 @SuppressWarnings({ "unchecked"})
 691                 TermTree<Feature> featureTree = repository.getTermTreeService().find(config.getFeatureTree());
 692                 filteredRootNode = filterFeatureNode(featureTree.getRoot(), existingFeatureUuids.keySet());
 693             } else {
 694                 filteredRootNode = createDefaultFeatureNode(taxon);
 695             }
 696
 697             //load facts per feature
 698             Map<UUID,Set<DescriptionElementBase>> featureMap = loadFeatureMap(taxon);
 699
 700             //load final result
 701             if (filteredRootNode != null && !filteredRootNode.getChildren().isEmpty()) {
 702                 ContainerDto<FeatureDto> features = new ContainerDto<>();
 703                 for (TreeNode<Feature,UUID> node : filteredRootNode.getChildren()) {
 704                     handleFeatureNode(config, featureMap, features, node, taxonPageDto);
 705                 }
 706                 taxonPageDto.setTaxonFacts(features);
 707             }
 708         } catch (Exception e) {
 709             //e.printStackTrace();
 710             taxonPageDto.addMessage(MessagesDto.NewErrorInstance("Error when loading factual data.", e));
 711         }
 712     }
 713
 714     //TODO merge with loadFacts, it is almost the same, see //DIFFERENT
 715     private void loadNameFacts(TaxonName name, TaxonBaseDto nameDto, TaxonPageDtoConfiguration config, TaxonPageDto pageDto) {
 716
 717         try {
 718             //compute the features that do exist for this taxon
 719             Map<UUID, Feature> existingFeatureUuids = getExistingFeatureUuids(name);
 720
 721             //filter, sort and structure according to feature tree
 722             TreeNode<Feature, UUID> filteredRootNode;
 723             //DIFFERENT
 724 //            if (config.getFeatureTree() != null) {
 725 //
 726 //                @SuppressWarnings({ "unchecked"})
 727 //                TermTree<Feature> featureTree = repository.getTermTreeService().find(config.getFeatureTree());
 728 //                filteredRootNode = filterFeatureNode(featureTree.getRoot(), existingFeatureUuids.keySet());
 729 //            } else {
 730                 filteredRootNode = createDefaultFeatureNode(name);
 731 //            }  //DIFFERENT END
 732
 733             //load facts per feature
 734             Map<UUID,Set<DescriptionElementBase>> featureMap = loadFeatureMap(name);
 735
 736             //load final result
 737             if (!filteredRootNode.getChildren().isEmpty()) {
 738                 ContainerDto<FeatureDto> features = new ContainerDto<>();
 739                 for (TreeNode<Feature,UUID> node : filteredRootNode.getChildren()) {
 740                     handleFeatureNode(config, featureMap, features, node, pageDto);
 741                 }
 742                 //DIFFERENT
 743                 nameDto.setNameFacts(features);
 744             }
 745         } catch (Exception e) {
 746             //e.printStackTrace();
 747             //DIFFERENT
 748             pageDto.addMessage(MessagesDto.NewErrorInstance("Error when loading factual data.", e));
 749         }
 750     }
 751
 752     private void handleFeatureNode(TaxonPageDtoConfiguration config,
 753             Map<UUID, Set<DescriptionElementBase>> featureMap, ContainerDto<FeatureDto> features,
 754             TreeNode<Feature, UUID> node, TaxonPageDto pageDto) {
 755
 756         Feature feature = node.getData();
 757         //TODO locale
 758         FeatureDto featureDto = new FeatureDto(feature.getUuid(), feature.getId(), feature.getLabel());
 759         features.addItem(featureDto);
 760
 761         List<Distribution> distributions = new ArrayList<>();
 762
 763         //
 764         for (DescriptionElementBase fact : featureMap.get(feature.getUuid())){
 765             if (fact.isInstanceOf(Distribution.class)) {
 766                 distributions.add(CdmBase.deproxy(fact, Distribution.class));
 767             }else {
 768                 //TODO how to handle CommonNames, do we also want to have a data structure
 769                 //with Language|
 770 //                             -- Area|
 771 //                                    --name
 772                 // a bit like for distribution??
 773                 handleFact(featureDto, fact, pageDto);
 774             }
 775         }
 776
 777         handleDistributions(config, featureDto, distributions, pageDto);
 778         //TODO really needed?
 779         orderFacts(featureDto);
 780
 781         //children
 782         ContainerDto<FeatureDto> childFeatures = new ContainerDto<>();
 783         for (TreeNode<Feature,UUID> child : node.getChildren()) {
 784             handleFeatureNode(config, featureMap, childFeatures, child, pageDto);
 785         }
 786         if (childFeatures.getCount() > 0) {
 787             featureDto.setSubFeatures(childFeatures);
 788         }
 789     }
 790
 791     //TODO not really used yet, only for distinguishing fact classes,
 792     //needs discussion if needed and how to implement.
 793     //we could also move compareTo methods to DTO classes but with this
 794     //remove from having only data in the DTO, no logic
 795     private void orderFacts(FeatureDto featureDto) {
 796         List<IFactDto> list = featureDto.getFacts().getItems();
 797         Collections.sort(list, (f1,f2)->{
 798             if (!f1.getClass().equals(f2.getClass())) {
 799                 return f1.getClass().getSimpleName().compareTo(f2.getClass().getSimpleName());
 800             }else {
 801                 if (f1 instanceof FactDto) {
 802                    FactDto fact1 = (FactDto)f1;
 803                    FactDto fact2 = (FactDto)f2;
 804
 805 //                   return fact1.getTypedLabel().toString().compareTo(fact2.getTypedLabel().toString());
 806                    return 0; //FIXME;
 807                 } else if (f1 instanceof CommonNameDto) {
 808                     int result = 0;
 809                     CommonNameDto fact1 = (CommonNameDto)f1;
 810                     CommonNameDto fact2 = (CommonNameDto)f2;
 811                     return 0;  //FIXME
 812                 }
 813             }
 814             return 0;  //FIXME
 815         });
 816
 817     }
 818
 819     private Map<UUID, Set<DescriptionElementBase>> loadFeatureMap(IDescribable<?> describable) {
 820         Map<UUID, Set<DescriptionElementBase>> featureMap = new HashMap<>();
 821
 822         //... load facts
 823         for (DescriptionBase<?> description : filterPublished(describable.getDescriptions())) {
 824             if (description.isImageGallery()) {
 825                 continue;
 826             }
 827             for (DescriptionElementBase deb : description.getElements()) {
 828                 Feature feature = deb.getFeature();
 829                 if (featureMap.get(feature.getUuid()) == null) {
 830                     featureMap.put(feature.getUuid(), new HashSet<>());
 831                 }
 832                 featureMap.get(feature.getUuid()).add(deb);
 833             }
 834         }
 835         return featureMap;
 836     }
 837
 838     private TreeNode<Feature, UUID> createDefaultFeatureNode(IDescribable<?> describable) {
 839         TreeNode<Feature, UUID> root = new TreeNode<>();
 840         Set<Feature> requiredFeatures = new HashSet<>();
 841
 842         for (DescriptionBase<?> description : describable.getDescriptions()) {
 843             if (description.isImageGallery()) {
 844                 continue;
 845             }
 846             for (DescriptionElementBase deb : description.getElements()) {
 847                 Feature feature = deb.getFeature();
 848                 if (feature != null) {  //null should not happen
 849                     requiredFeatures.add(feature);
 850                 }
 851             }
 852         }
 853         List<Feature> sortedChildren = new ArrayList<>(requiredFeatures);
 854         Collections.sort(sortedChildren, (f1,f2) -> f1.getTitleCache().compareTo(f2.getTitleCache()));
 855         sortedChildren.stream().forEachOrdered(f->root.addChild(new TreeNode<>(f.getUuid(), f)));
 856         return root;
 857     }
 858
 859     /**
 860      * Recursive call to a feature tree's feature node in order to creates a tree structure
 861      * ordered in the same way as the according feature tree but only containing features
 862      * that do really exist for the given taxon. If only a child node is required the parent
 863      * node/feature is also considered to be required.<BR>
 864      */
 865     private TreeNode<Feature, UUID> filterFeatureNode(TermNode<Feature> featureNode,
 866             Set<UUID> existingFeatureUuids) {
 867
 868         //first filter children
 869         List<TreeNode<Feature, UUID>> requiredChildNodes = new ArrayList<>();
 870         for (TermNode<Feature> childNode : featureNode.getChildNodes()) {
 871             TreeNode<Feature, UUID> child = filterFeatureNode(childNode, existingFeatureUuids);
 872             if (child != null) {
 873                 requiredChildNodes.add(child);
 874             }
 875         }
 876
 877         //if any child is required or this node is required ....
 878         if (!requiredChildNodes.isEmpty() ||
 879                 featureNode.getTerm() != null && existingFeatureUuids.contains(featureNode.getTerm().getUuid())) {
 880
 881             TreeNode<Feature,UUID> result = new TreeNode<>();
 882             //add this nodes data
 883             Feature feature = featureNode.getTerm() == null ? null : featureNode.getTerm();
 884             if (feature != null) {
 885                 result.setNodeId(feature.getUuid());
 886                 result.setData(feature);
 887             }
 888             //add child data
 889             requiredChildNodes.stream().forEachOrdered(c->result.addChild(c));
 890             return result;
 891         }else {
 892             return null;
 893         }
 894     }
 895
 896     /**
 897      * Computes the (unsorted) set of features for  which facts exist
 898      * for the given taxon.
 899      */
 900     private Map<UUID, Feature> getExistingFeatureUuids(IDescribable<?> describable) {
 901         Map<UUID, Feature> result = new HashMap<>();
 902         for (DescriptionBase<?> description : filterPublished(describable.getDescriptions())) {
 903             if (description.isImageGallery()) {
 904                 continue;
 905             }
 906             for (DescriptionElementBase deb : description.getElements()) {
 907                 Feature feature = deb.getFeature();
 908                 if (feature != null) {  //null should not happen
 909                     result.put(feature.getUuid(), feature);
 910                 }
 911             }
 912         }
 913         return result;
 914     }
 915
 916     private void handleDistributions(TaxonPageDtoConfiguration config, FeatureDto featureDto,
 917             List<Distribution> distributions, TaxonPageDto pageDto) {
 918
 919         if (distributions.isEmpty()) {
 920             return;
 921         }
 922         IDistributionService distributionService = repository.getDistributionService();
 923
 924         //configs
 925         DistributionInfoConfiguration distributionConfig = config.getDistributionInfoConfiguration(featureDto.getUuid());
 926         CondensedDistributionConfiguration condensedConfig = distributionConfig.getCondensedDistributionConfiguration();
 927
 928         String statusColorsString = distributionConfig.getStatusColorsString();
 929
 930
 931         //copied from DescriptionListController
 932
 933         boolean ignoreDistributionStatusUndefined = true;  //workaround until #9500 is fully implemented
 934         distributionConfig.setIgnoreDistributionStatusUndefined(ignoreDistributionStatusUndefined);
 935         boolean neverUseFallbackAreaAsParent = true;  //may become a service parameter in future
 936
 937         //fallbackArea markers include markers for fully hidden areas and fallback areas.
 938         //The later are hidden markers on areas that have non-hidden subareas (#4408)
 939         Set<MarkerType> fallbackAreaMarkerTypes = distributionConfig.getFallbackAreaMarkerTypeList();
 940         if(!CdmUtils.isNullSafeEmpty(fallbackAreaMarkerTypes)){
 941             condensedConfig.fallbackAreaMarkers = fallbackAreaMarkerTypes.stream().map(mt->mt.getUuid()).collect(Collectors.toSet());
 942         }
 943
 944         Map<PresenceAbsenceTerm, Color> distributionStatusColors;
 945         try {
 946             distributionStatusColors = DistributionServiceUtilities.buildStatusColorMap(
 947                     statusColorsString, repository.getTermService(), repository.getVocabularyService());
 948         } catch (JsonProcessingException e) {
 949             pageDto.addMessage(MessagesDto.NewErrorInstance("JsonProcessingException when reading distribution status colors", e));
 950             //TODO is null allowed?
 951             distributionStatusColors = null;
 952         }
 953
 954         DistributionInfoDto dto = distributionService.composeDistributionInfoFor(distributionConfig, distributions,
 955                 neverUseFallbackAreaAsParent,
 956                 distributionStatusColors, LocaleContext.getLanguages());
 957
 958         //should not be necessary anymore as distribution data is now loaded directly in DIstributionServiceImpl
 959         //by calling PortalDtoLoader.loadBaseData() directly
 960 //        if (distributionConfig.isUseTreeDto() && dto.getTree() != null) {
 961 //            DistributionTreeDto tree = (DistributionTreeDto)dto.getTree();
 962 //            TreeNode<Set<DistributionDto>, NamedAreaDto> root = tree.getRootElement();
 963 //            //fill uuid->distribution map
 964 //            Map<UUID,Distribution> distributionMap = new HashMap<>();
 965 //            distributions.stream().forEach(d->distributionMap.put(d.getUuid(), d));
 966 //            handleDistributionDtoNode(distributionMap, root);
 967 //        }
 968
 969         featureDto.addFact(dto);
 970     }
 971
 972     //should not be necessary anymore as distribution data is now loaded directly in DIstributionServiceImpl
 973     //by calling PortalDtoLoader.loadBaseData() directly
 974 //    private static void handleDistributionDtoNode(Map<UUID, Distribution> map,
 975 //            TreeNode<Set<DistributionDto>, NamedAreaDto> root) {
 976 //       if (root.getData() != null) {
 977 //           root.getData().stream().forEach(dto->{
 978 //               Distribution distr  = map.get(dto.getUuid());
 979 //               loadBaseData(distr, dto);
 980 //               dto.setTimeperiod(distr.getTimeperiod() == null ? null : distr.getTimeperiod().toString());
 981 //           });
 982 //       }
 983 //
 984 //       //handle children
 985 //       if (root.getChildren() != null) {
 986 //           root.getChildren().stream().forEach(c->handleDistributionDtoNode(map, c));
 987 //       }
 988 //    }
 989
 990     private FactDtoBase handleFact(FeatureDto featureDto, DescriptionElementBase fact, TaxonPageDto pageDto) {
 991         //TODO locale
 992         Language localeLang = null;
 993
 994         FactDtoBase result;
 995         if (fact.isInstanceOf(TextData.class)) {
 996             TextData td = CdmBase.deproxy(fact, TextData.class);
 997             LanguageString ls = td.getPreferredLanguageString(localeLang);
 998             String text = ls == null ? "" : CdmUtils.Nz(ls.getText());
 999
1000             FactDto factDto = new FactDto();
1001             featureDto.addFact(factDto);
1002             //TODO do we really need type information for textdata here?
1003             TypedLabel typedLabel = new TypedLabel(text);
1004             typedLabel.setClassAndId(td);
1005             factDto.getTypedLabel().add(typedLabel);
1006             loadBaseData(td, factDto);
1007             //TODO
1008             result = factDto;
1009         }else if (fact.isInstanceOf(CommonTaxonName.class)) {
1010             CommonTaxonName ctn = CdmBase.deproxy(fact, CommonTaxonName.class);
1011             CommonNameDto dto = new CommonNameDto();
1012             featureDto.addFact(dto);
1013
1014             Language lang = ctn.getLanguage();
1015             if (lang != null) {
1016                 String langLabel = getTermLabel(lang, localeLang);
1017                 dto.setLanguage(langLabel);
1018                 dto.setLanguageUuid(lang.getUuid());
1019             }else {
1020                 //TODO
1021                 dto.setLanguage("-");
1022             }
1023             //area
1024             NamedArea area = ctn.getArea();
1025             if (area != null) {
1026                 String areaLabel = getTermLabel(area, localeLang);
1027                 dto.setArea(areaLabel);
1028                 dto.setAreaUUID(area.getUuid());
1029             }
1030             dto.setName(ctn.getName());
1031             loadBaseData(ctn, dto);
1032             //TODO sort all common names (not urgent as this is done by portal code)
1033             result = dto;
1034         } else if (fact.isInstanceOf(IndividualsAssociation.class)) {
1035             IndividualsAssociation ia = CdmBase.deproxy(fact, IndividualsAssociation.class);
1036             IndividualsAssociationDto dto = new IndividualsAssociationDto ();
1037
1038             LanguageString description = MultilanguageTextHelper.getPreferredLanguageString(ia.getDescription(), Arrays.asList(localeLang));
1039             if (description != null) {
1040                 dto.setDescritpion(description.getText());
1041             }
1042             SpecimenOrObservationBase<?> specimen = ia.getAssociatedSpecimenOrObservation();
1043             if (specimen != null) {
1044                 //TODO what to use here??
1045                 dto.setOccurrence(specimen.getTitleCache());
1046                 dto.setOccurrenceUuid(specimen.getUuid());
1047             }
1048
1049             featureDto.addFact(dto);
1050             loadBaseData(ia, dto);
1051             result = dto;
1052         } else if (fact.isInstanceOf(TaxonInteraction.class)) {
1053             TaxonInteraction ti = CdmBase.deproxy(fact, TaxonInteraction.class);
1054             TaxonInteractionDto dto = new TaxonInteractionDto ();
1055
1056             LanguageString description = MultilanguageTextHelper.getPreferredLanguageString(
1057                     ti.getDescription(), Arrays.asList(localeLang));
1058             if (description != null) {
1059                 dto.setDescritpion(description.getText());
1060             }
1061             Taxon taxon = ti.getTaxon2();
1062             if (taxon != null) {
1063                 //TODO what to use here??
1064                 dto.setTaxon(taxon.cacheStrategy().getTaggedTitle(taxon));
1065                 dto.setTaxonUuid(taxon.getUuid());
1066             }
1067             featureDto.addFact(dto);
1068             loadBaseData(ti, dto);
1069             result = dto;
1070         }else if (fact.isInstanceOf(CategoricalData.class)) {
1071             CategoricalData cd = CdmBase.deproxy(fact, CategoricalData.class);
1072             FactDto factDto = new FactDto();
1073             featureDto.addFact(factDto);
1074             //TODO do we really need type information for textdata here?
1075             String label = CategoricalDataFormatter.NewInstance(null).format(cd, localeLang);
1076             TypedLabel typedLabel = new TypedLabel(label);
1077             typedLabel.setClassAndId(cd);
1078             factDto.getTypedLabel().add(typedLabel);
1079             //TODO
1080             loadBaseData(cd, factDto);
1081             result = factDto;
1082         }else if (fact.isInstanceOf(QuantitativeData.class)) {
1083             QuantitativeData qd = CdmBase.deproxy(fact, QuantitativeData.class);
1084             FactDto factDto = new FactDto();
1085             featureDto.addFact(factDto);
1086             //TODO do we really need type information for textdata here?
1087             String label = QuantitativeDataFormatter.NewInstance(null).format(qd, localeLang);
1088             TypedLabel typedLabel = new TypedLabel(label);
1089             typedLabel.setClassAndId(qd);
1090             factDto.getTypedLabel().add(typedLabel);
1091             //TODO
1092             loadBaseData(qd, factDto);
1093             result = factDto;
1094         }else if (fact.isInstanceOf(TemporalData.class)) {
1095             TemporalData td = CdmBase.deproxy(fact, TemporalData.class);
1096             FactDto factDto = new FactDto();
1097             featureDto.addFact(factDto);
1098             //TODO do we really need type information for textdata here?
1099             String label = td.toString();
1100             TypedLabel typedLabel = new TypedLabel(label);
1101             typedLabel.setClassAndId(td);
1102             factDto.getTypedLabel().add(typedLabel);
1103             //TODO
1104             loadBaseData(td, factDto);
1105             result = factDto;
1106         }else {
1107             pageDto.addMessage(MessagesDto.NewWarnInstance("DescriptionElement type not yet handled: " + fact.getClass().getSimpleName()));
1108             return null;
1109         }
1110         result.setTimeperiod(fact.getTimeperiod() == null ? null : fact.getTimeperiod().toString());
1111         return result;
1112     }
1113
1114     private String getTermLabel(TermBase term, Language localeLang) {
1115         if (term == null) {
1116             return null;
1117         }
1118         Representation rep = term.getPreferredRepresentation(localeLang);
1119         String label = rep == null ? null : rep.getLabel();
1120         label = label == null ? term.getLabel() : label;
1121         return label;
1122     }
1123
1124     /**
1125      * Compares an existing last date and the last date of an entity
1126      * and returns the resulting last date.
1127      */
1128     private LocalDateTime getLastUpdated(LocalDateTime existingLastDate, VersionableEntity dateToAddEntity) {
1129
1130         DateTime dateToAdd = dateToAddEntity.getUpdated() != null ? dateToAddEntity.getUpdated() : dateToAddEntity.getCreated();
1131
1132         LocalDateTime javaLocalDateTimeOfEntity = dateToAdd == null ? null:
1133                 LocalDateTime.of(dateToAdd.getYear(), dateToAdd.getMonthOfYear(),
1134                         dateToAdd.getDayOfMonth(), dateToAdd.getHourOfDay(),
1135                         dateToAdd.getMinuteOfHour(), dateToAdd.getSecondOfMinute());
1136
1137        if (existingLastDate == null) {
1138            return javaLocalDateTimeOfEntity;
1139        }else if (javaLocalDateTimeOfEntity == null || javaLocalDateTimeOfEntity.compareTo(existingLastDate) < 0)  {
1140            return existingLastDate;
1141        }else {
1142            return javaLocalDateTimeOfEntity;
1143        }
1144     }
1145
1146     public static void loadBaseData(CdmBase cdmBase, CdmBaseDto dto) {
1147         dto.setId(cdmBase.getId());
1148         dto.setUuid(cdmBase.getUuid());
1149
1150         loadAnnotatable(cdmBase, dto);
1151         loadSources(cdmBase, dto);
1152         //loadIdentifiable(cdmBase, dto);
1153     }
1154
1155     private static void loadSources(CdmBase cdmBase, CdmBaseDto dto) {
1156         if (dto instanceof SingleSourcedDto && cdmBase.isInstanceOf(SingleSourcedEntityBase.class)) {
1157             //TODO other sourced
1158             SingleSourcedEntityBase sourced = CdmBase.deproxy(cdmBase, SingleSourcedEntityBase.class);
1159             SingleSourcedDto sourcedDto = (SingleSourcedDto)dto;
1160             NamedSource source = sourced.getSource();
1161             if (source != null) { //TODO  && !source.isEmpty() - does not exist yet
1162                 SourceDto sourceDto = makeSource(source);
1163                 sourcedDto.setSource(sourceDto);
1164             }
1165         } else if (dto instanceof SourcedDto && cdmBase instanceof ISourceable) {
1166             @SuppressWarnings("unchecked")
1167             ISourceable<OriginalSourceBase> sourced = (ISourceable<OriginalSourceBase>)cdmBase;
1168             SourcedDto sourcedDto = (SourcedDto)dto;
1169             for (OriginalSourceBase source : sourced.getSources()) {
1170                 SourceDto sourceDto = makeSource(source);
1171                 sourcedDto.addSource(sourceDto);
1172             }
1173         }
1174         //load description sources for facts
1175         if (cdmBase.isInstanceOf(DescriptionElementBase.class)){
1176             DescriptionBase<?> db = CdmBase.deproxy(cdmBase, DescriptionElementBase.class).getInDescription();
1177             if (db != null) {  //test sometime do not have a description for facts
1178                 SourcedDto sourcedDto = (SourcedDto)dto;
1179                 for (OriginalSourceBase source : db.getSources()) {
1180                     SourceDto sourceDto = new SourceDto();
1181                     loadSource(source, sourceDto);
1182                     sourcedDto.addSource(sourceDto);
1183                 }
1184             }
1185         }
1186     }
1187
1188     private static SourceDto makeSource(OriginalSourceBase source) {
1189         if (source == null) {
1190             return null;
1191         }
1192         SourceDto sourceDto = new SourceDto();
1193         loadSource(source, sourceDto);
1194         return sourceDto;
1195     }
1196
1197     private static void loadSource(OriginalSourceBase source, SourceDto sourceDto) {
1198
1199         source = CdmBase.deproxy(source);
1200         //base data
1201         loadBaseData(source, sourceDto);
1202
1203         ICdmBase linkedObject = source.getCitation();
1204         if (linkedObject == null) {
1205             //cdmsource
1206             linkedObject = source.getCdmSource();
1207         }
1208
1209         //citation doi & uri & links
1210         Reference ref = source.getCitation();
1211         if (ref != null) {
1212             sourceDto.setDoi(ref.getDoiString());
1213             sourceDto.setUri(ref.getUri());
1214             sourceDto.setUuid(ref.getUuid());
1215             sourceDto.setOriginalInfo(source.getOriginalInfo());
1216             Set<ExternalLink> links = ref.getLinks();
1217             for (ExternalLink link : links) {
1218                 if (link.getUri() != null) {
1219                     sourceDto.addLink(link.getUri());
1220                 }
1221             }
1222         }
1223
1224         //label
1225         //TODO this probably does not use specimen or cdmSource if necessary,
1226         //     also long citation is still preliminary
1227         String label = OriginalSourceFormatter.INSTANCE_LONG_CITATION.format(source);
1228         TypedLabel typedLabel = new TypedLabel(source, label);
1229         sourceDto.addLabel(typedLabel);
1230         sourceDto.setType(source.getType() != null ? source.getType().toString() : null);
1231
1232         if (source.isInstanceOf(NamedSourceBase.class)) {
1233             NamedSourceBase ns = CdmBase.deproxy(source, NamedSourceBase.class);
1234
1235             //nameUsedInSource
1236             TaxonName name =  ns.getNameUsedInSource();
1237             if (name != null) {
1238                 List<TaggedText> taggedName = name.cacheStrategy().getTaggedTitle(name);
1239                 //TODO nom status?
1240                 sourceDto.setNameInSource(taggedName);
1241                 sourceDto.setNameInSourceUuid(name.getUuid());
1242             }
1243
1244             //specimen uuid
1245             if (source.isInstanceOf(DescriptionElementSource.class)) {
1246                 DescriptionElementSource des = CdmBase.deproxy(source, DescriptionElementSource.class);
1247                 if (linkedObject == null) {
1248                     linkedObject = des.getSpecimen();
1249                 }
1250             }
1251         }
1252
1253         sourceDto.setLinkedUuid(getUuid(linkedObject));
1254         String linkedObjectStr = linkedObject == null ? null : CdmBase.deproxy(linkedObject).getClass().getSimpleName();
1255         sourceDto.setLinkedClass(linkedObjectStr);
1256     }
1257
1258     private static UUID getUuid(ICdmBase cdmBase) {
1259         return cdmBase == null ? null : cdmBase.getUuid();
1260     }
1261
1262     private static void loadAnnotatable(CdmBase cdmBase, CdmBaseDto dto) {
1263         if (dto instanceof AnnotatableDto && cdmBase.isInstanceOf(AnnotatableEntity.class)) {
1264             AnnotatableEntity annotatable = CdmBase.deproxy(cdmBase, AnnotatableEntity.class);
1265             AnnotatableDto annotatableDto = (AnnotatableDto)dto;
1266             //annotation
1267             for (Annotation annotation : annotatable.getAnnotations()) {
1268                 if (annotation.getAnnotationType() != null
1269                         //TODO annotation type filter
1270                         && annotation.getAnnotationType().getUuid().equals(AnnotationType.uuidEditorial)
1271                         && StringUtils.isNotBlank(annotation.getText())) {
1272
1273                     AnnotationDto annotationDto = new AnnotationDto();
1274                     annotatableDto.addAnnotation(annotationDto);
1275                     //TODO id needed? but need to adapt dto and container then
1276                     loadBaseData(annotation, annotationDto);
1277                     annotationDto.setText(annotation.getText());
1278                     UUID uuidAnnotationType = annotation.getAnnotationType() == null ? null :annotation.getAnnotationType().getUuid();
1279                     annotationDto.setTypeUuid(uuidAnnotationType);
1280                     //language etc. currently not yet used
1281                 }
1282             }
1283
1284             //marker
1285             for (Marker marker : annotatable.getMarkers()) {
1286                 if (marker.getMarkerType() != null
1287                         //TODO markertype filter
1288 //                        && marker.getMarkerType().getUuid().equals(AnnotationType.uuidEditorial)
1289                            ){
1290
1291                     MarkerDto markerDto = new MarkerDto();
1292                     annotatableDto.addMarker(markerDto);
1293                     //TODO id needed? but need to adapt dto and container then
1294                     loadBaseData(marker, markerDto);
1295                     if (marker.getMarkerType() != null) {
1296                         markerDto.setTypeUuid(marker.getMarkerType().getUuid());
1297                         //TODO locale
1298                         markerDto.setType(marker.getMarkerType().getTitleCache());
1299                     }
1300                     markerDto.setValue(marker.getValue());
1301                 }
1302             }
1303         }
1304     }
1305 }