cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/OccurrenceServiceImpl.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9 package eu.etaxonomy.cdm.api.service;
  10
  11 import java.io.IOException;
  12 import java.util.ArrayList;
  13 import java.util.Arrays;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.Comparator;
  17 import java.util.EnumSet;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24 import java.util.UUID;
  25 import java.util.stream.Collectors;
  26
  27 import org.apache.commons.lang3.StringUtils;
  28 import org.apache.logging.log4j.LogManager;
  29 import org.apache.logging.log4j.Logger;
  30 import org.apache.lucene.queryparser.classic.ParseException;
  31 import org.apache.lucene.search.BooleanClause.Occur;
  32 import org.apache.lucene.search.BooleanQuery.Builder;
  33 import org.apache.lucene.search.SortField;
  34 import org.apache.lucene.search.grouping.TopGroups;
  35 import org.apache.lucene.util.BytesRef;
  36 import org.hibernate.TransientObjectException;
  37 import org.springframework.beans.factory.annotation.Autowired;
  38 import org.springframework.dao.DataRetrievalFailureException;
  39 import org.springframework.stereotype.Service;
  40 import org.springframework.transaction.annotation.Transactional;
  41
  42 import eu.etaxonomy.cdm.api.service.UpdateResult.Status;
  43 import eu.etaxonomy.cdm.api.service.config.DeleteConfiguratorBase;
  44 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  45 import eu.etaxonomy.cdm.api.service.config.IIdentifiableEntityServiceConfigurator;
  46 import eu.etaxonomy.cdm.api.service.config.SpecimenDeleteConfigurator;
  47 import eu.etaxonomy.cdm.api.service.dto.DNASampleDTO;
  48 import eu.etaxonomy.cdm.api.service.dto.DerivedUnitDTO;
  49 import eu.etaxonomy.cdm.api.service.dto.FieldUnitDTO;
  50 import eu.etaxonomy.cdm.api.service.dto.MediaDTO;
  51 import eu.etaxonomy.cdm.api.service.dto.RectangleDTO;
  52 import eu.etaxonomy.cdm.api.service.dto.SpecimenOrObservationBaseDTO;
  53 import eu.etaxonomy.cdm.api.service.dto.SpecimenOrObservationDTOFactory;
  54 import eu.etaxonomy.cdm.api.service.exception.ReferencedObjectUndeletableException;
  55 import eu.etaxonomy.cdm.api.service.molecular.ISequenceService;
  56 import eu.etaxonomy.cdm.api.service.pager.Pager;
  57 import eu.etaxonomy.cdm.api.service.pager.impl.AbstractPagerImpl;
  58 import eu.etaxonomy.cdm.api.service.pager.impl.DefaultPagerImpl;
  59 import eu.etaxonomy.cdm.api.service.search.ILuceneIndexToolProvider;
  60 import eu.etaxonomy.cdm.api.service.search.ISearchResultBuilder;
  61 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  62 import eu.etaxonomy.cdm.api.service.search.LuceneSearch;
  63 import eu.etaxonomy.cdm.api.service.search.QueryFactory;
  64 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  65 import eu.etaxonomy.cdm.api.service.search.SearchResultBuilder;
  66 import eu.etaxonomy.cdm.api.util.TaxonRelationshipEdge;
  67 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  68 import eu.etaxonomy.cdm.compare.common.PartialComparator;
  69 import eu.etaxonomy.cdm.facade.DerivedUnitFacade;
  70 import eu.etaxonomy.cdm.facade.DerivedUnitFacadeConfigurator;
  71 import eu.etaxonomy.cdm.facade.DerivedUnitFacadeNotSupportedException;
  72 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  73 import eu.etaxonomy.cdm.model.CdmBaseType;
  74 import eu.etaxonomy.cdm.model.common.CdmBase;
  75 import eu.etaxonomy.cdm.model.common.Language;
  76 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  77 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  78 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  79 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  80 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  81 import eu.etaxonomy.cdm.model.location.Point;
  82 import eu.etaxonomy.cdm.model.media.Media;
  83 import eu.etaxonomy.cdm.model.molecular.AmplificationResult;
  84 import eu.etaxonomy.cdm.model.molecular.DnaSample;
  85 import eu.etaxonomy.cdm.model.molecular.Sequence;
  86 import eu.etaxonomy.cdm.model.molecular.SingleRead;
  87 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  88 import eu.etaxonomy.cdm.model.name.TaxonName;
  89 import eu.etaxonomy.cdm.model.name.TypeDesignationBase;
  90 import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
  91 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  92 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  93 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  94 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  95 import eu.etaxonomy.cdm.model.occurrence.MediaSpecimen;
  96 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  97 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  98 import eu.etaxonomy.cdm.model.taxon.Taxon;
  99 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
 100 import eu.etaxonomy.cdm.persistence.dao.initializer.AbstractBeanInitializer;
 101 import eu.etaxonomy.cdm.persistence.dao.occurrence.IOccurrenceDao;
 102 import eu.etaxonomy.cdm.persistence.dto.SpecimenNodeWrapper;
 103 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
 104 import eu.etaxonomy.cdm.persistence.query.AssignmentStatus;
 105 import eu.etaxonomy.cdm.persistence.query.OrderHint;
 106 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
 107
 108 /**
 109  * @author a.babadshanjan
 110  * @since 01.09.2008
 111  */
 112 @Service
 113 @Transactional(readOnly = true)
 114 public class OccurrenceServiceImpl
 115         extends IdentifiableServiceBase<SpecimenOrObservationBase, IOccurrenceDao>
 116         implements IOccurrenceService {
 117
 118     private static final Logger logger = LogManager.getLogger();
 119
 120     @Autowired
 121     private IDescriptionService descriptionService;
 122
 123     @Autowired
 124     private INameService nameService;
 125
 126     @Autowired
 127     private IEventBaseService eventService;
 128
 129     @Autowired
 130     private ITaxonService taxonService;
 131
 132     @Autowired
 133     private ISequenceService sequenceService;
 134
 135     @Autowired
 136     private AbstractBeanInitializer<?> beanInitializer;
 137
 138     @Autowired
 139     private ILuceneIndexToolProvider luceneIndexToolProvider;
 140
 141     public OccurrenceServiceImpl() {
 142         logger.debug("Load OccurrenceService Bean");
 143     }
 144
 145     @Override
 146     @Transactional(readOnly = false)
 147     public UpdateResult updateCaches(Class<? extends SpecimenOrObservationBase> clazz, Integer stepSize, IIdentifiableEntityCacheStrategy<SpecimenOrObservationBase> cacheStrategy, IProgressMonitor monitor) {
 148         if (clazz == null) {
 149             clazz = SpecimenOrObservationBase.class;
 150         }
 151         return super.updateCachesImpl(clazz, stepSize, cacheStrategy, monitor);
 152     }
 153
 154     @Override
 155     @Autowired
 156     protected void setDao(IOccurrenceDao dao) {
 157         this.dao = dao;
 158     }
 159
 160     @Override
 161     public Pager<DerivationEvent> getDerivationEvents(SpecimenOrObservationBase occurence, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 162         long numberOfResults = dao.countDerivationEvents(occurence);
 163
 164         List<DerivationEvent> results = new ArrayList<>();
 165         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 166             results = dao.getDerivationEvents(occurence, pageSize, pageNumber, propertyPaths);
 167         }
 168
 169         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 170     }
 171
 172     @Override
 173     public long countDeterminations(SpecimenOrObservationBase occurence, TaxonBase taxonbase) {
 174         return dao.countDeterminations(occurence, taxonbase);
 175     }
 176
 177     @Override
 178     public Pager<DeterminationEvent> getDeterminations(SpecimenOrObservationBase occurrence, TaxonBase taxonBase,
 179             Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 180         long numberOfResults = dao.countDeterminations(occurrence, taxonBase);
 181
 182         List<DeterminationEvent> results = new ArrayList<>();
 183         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 184             results = dao.getDeterminations(occurrence, taxonBase, pageSize, pageNumber, propertyPaths);
 185         }
 186
 187         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 188     }
 189
 190     @Override
 191     public Pager<Media> getMedia(SpecimenOrObservationBase occurence, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 192         long numberOfResults = dao.countMedia(occurence);
 193
 194         List<Media> results = new ArrayList<>();
 195         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 196             results = dao.getMedia(occurence, pageSize, pageNumber, propertyPaths);
 197         }
 198
 199         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 200     }
 201
 202     @Override
 203     public Pager<MediaDTO> getMediaDTOs(SpecimenOrObservationBase<?> occurence, Integer pageSize, Integer pageNumber) {
 204         long numberOfResults = dao.countMedia(occurence);
 205
 206         List<Media> results = new ArrayList<>();
 207         if(AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)) {
 208             results = dao.getMedia(occurence, pageSize, pageNumber, null);
 209         }
 210         List<MediaDTO> mediaDTOs = results.stream()
 211                 .map(m -> MediaDTO.fromEntity(m))
 212                 .flatMap(dtos -> dtos.stream())
 213                 .collect(Collectors.toList()
 214                 );
 215         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, mediaDTOs);
 216     }
 217     @Override
 218     public Pager<Media> getMediaInHierarchy(SpecimenOrObservationBase<?> rootOccurence, boolean collectOriginalMedia, boolean collectDerivateMedia, Integer pageSize,
 219             Integer pageNumber, List<String> propertyPaths) {
 220          List<Media> media = new ArrayList<>();
 221          //media specimens
 222          if(rootOccurence.isInstanceOf(MediaSpecimen.class)){
 223              MediaSpecimen mediaSpecimen = HibernateProxyHelper.deproxy(rootOccurence, MediaSpecimen.class);
 224              media.add(mediaSpecimen.getMediaSpecimen());
 225          }
 226          // pherograms & gelPhotos
 227          if (rootOccurence.isInstanceOf(DnaSample.class)) {
 228              DnaSample dnaSample = CdmBase.deproxy(rootOccurence, DnaSample.class);
 229              Set<Sequence> sequences = dnaSample.getSequences();
 230              //we do show only those gelPhotos which lead to a consensus sequence
 231              for (Sequence sequence : sequences) {
 232                  Set<Media> dnaRelatedMedia = new HashSet<>();
 233                  for (SingleRead singleRead : sequence.getSingleReads()){
 234                      AmplificationResult amplification = singleRead.getAmplificationResult();
 235                      dnaRelatedMedia.add(amplification.getGelPhoto());
 236                      dnaRelatedMedia.add(singleRead.getPherogram());
 237                      dnaRelatedMedia.remove(null);
 238                  }
 239                  media.addAll(dnaRelatedMedia);
 240              }
 241          }
 242          if(rootOccurence.isInstanceOf(DerivedUnit.class)){
 243              DerivedUnit derivedUnit = HibernateProxyHelper.deproxy(rootOccurence, DerivedUnit.class);
 244              if (collectDerivateMedia) {
 245                      for (DerivationEvent derivationEvent : derivedUnit.getDerivationEvents()) {
 246                          for (DerivedUnit childDerivative : derivationEvent.getDerivatives()) {
 247                                  //collectOriginalMedia should only called for the first derived unit
 248                              media.addAll(getMediaInHierarchy(childDerivative, false, true, pageSize, pageNumber, propertyPaths).getRecords());
 249                          }
 250                      }
 251              }
 252              if (collectOriginalMedia) {
 253                  for (SpecimenOrObservationBase original : derivedUnit.getOriginals()) {
 254                                  //collect media to the top of the tree
 255                              media.addAll(getMediaInHierarchy(original, true, false, pageSize, pageNumber, propertyPaths).getRecords());
 256                      }
 257                  }
 258          }
 259
 260
 261          return new DefaultPagerImpl<>(pageNumber, Long.valueOf(media.size()), pageSize, media);
 262     }
 263
 264
 265     @Override
 266     public Pager<Media> getMediaInHierarchy(SpecimenOrObservationBase<?> rootOccurence, Integer pageSize,
 267             Integer pageNumber, List<String> propertyPaths) {
 268         return getMediaInHierarchy(rootOccurence, false, true, pageSize,
 269                 pageNumber, propertyPaths);
 270
 271     }
 272
 273     @Override
 274     public Pager<SpecimenOrObservationBase> list(Class<? extends SpecimenOrObservationBase> type, TaxonName determinedAs,
 275             Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 276         long numberOfResults = dao.count(type, determinedAs);
 277         @SuppressWarnings("rawtypes")
 278         List<SpecimenOrObservationBase> results = new ArrayList<>();
 279         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 280             results = dao.list(type, determinedAs, pageSize, pageNumber, orderHints, propertyPaths);
 281         }
 282         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 283     }
 284
 285     @Override
 286     public Pager<SpecimenOrObservationBase> list(Class<? extends SpecimenOrObservationBase> type, TaxonBase determinedAs,
 287             Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 288         long numberOfResults = dao.count(type, determinedAs);
 289         @SuppressWarnings("rawtypes")
 290         List<SpecimenOrObservationBase> results = new ArrayList<>();
 291         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 292             results = dao.list(type, determinedAs, pageSize, pageNumber, orderHints, propertyPaths);
 293         }
 294         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 295     }
 296
 297     @Override
 298     public List<UuidAndTitleCache<DerivedUnit>> getDerivedUnitUuidAndTitleCache(Integer limit, String pattern) {
 299         return dao.getDerivedUnitUuidAndTitleCache(limit, pattern);
 300     }
 301
 302     @Override
 303     public List<UuidAndTitleCache<FieldUnit>> getFieldUnitUuidAndTitleCache() {
 304         return dao.getFieldUnitUuidAndTitleCache();
 305     }
 306
 307     @Override
 308     public DerivedUnitFacade getDerivedUnitFacade(DerivedUnit derivedUnit, List<String> derivedUnitFacadeInitStrategy) throws DerivedUnitFacadeNotSupportedException {
 309         derivedUnit = (DerivedUnit) dao.load(derivedUnit.getUuid(), null);
 310         DerivedUnitFacadeConfigurator config = DerivedUnitFacadeConfigurator.NewInstance();
 311         config.setThrowExceptionForNonSpecimenPreservationMethodRequest(false);
 312         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(derivedUnit, config);
 313         beanInitializer.initialize(derivedUnitFacade, derivedUnitFacadeInitStrategy);
 314         return derivedUnitFacade;
 315     }
 316
 317     @Override
 318     public List<DerivedUnitFacade> listDerivedUnitFacades(
 319             DescriptionBase description, List<String> derivedUnitFacadeInitStrategy) {
 320
 321         List<DerivedUnitFacade> derivedUnitFacadeList = new ArrayList<>();
 322         IndividualsAssociation tempIndividualsAssociation;
 323         SpecimenOrObservationBase tempSpecimenOrObservationBase;
 324         List<IndividualsAssociation> elements = descriptionService.listDescriptionElements(description, null, IndividualsAssociation.class, null, 0, Arrays.asList(new String []{"associatedSpecimenOrObservation"}));
 325         for (IndividualsAssociation element : elements) {
 326             tempIndividualsAssociation = HibernateProxyHelper.deproxy(element, IndividualsAssociation.class);
 327             if (tempIndividualsAssociation.getAssociatedSpecimenOrObservation() != null) {
 328                 tempSpecimenOrObservationBase = HibernateProxyHelper.deproxy(tempIndividualsAssociation.getAssociatedSpecimenOrObservation(), SpecimenOrObservationBase.class);
 329                 if (tempSpecimenOrObservationBase.isInstanceOf(DerivedUnit.class)) {
 330                     try {
 331                         derivedUnitFacadeList.add(DerivedUnitFacade.NewInstance(HibernateProxyHelper.deproxy(tempSpecimenOrObservationBase, DerivedUnit.class)));
 332                     } catch (DerivedUnitFacadeNotSupportedException e) {
 333                         logger.warn(tempIndividualsAssociation.getAssociatedSpecimenOrObservation().getTitleCache() + " : " + e.getMessage());
 334                     }
 335                 }
 336             }
 337         }
 338
 339         beanInitializer.initializeAll(derivedUnitFacadeList, derivedUnitFacadeInitStrategy);
 340
 341         return derivedUnitFacadeList;
 342     }
 343
 344
 345     @Override
 346     public <T extends SpecimenOrObservationBase> List<T> listByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 347             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 348
 349         return pageByAssociatedTaxon(type, includeRelationships, associatedTaxon, maxDepth, pageSize, pageNumber, orderHints, propertyPaths).getRecords();
 350     }
 351
 352     @Override
 353     public Collection<SpecimenNodeWrapper> listUuidAndTitleCacheByAssociatedTaxon(List<UUID> taxonNodeUuids,
 354             Integer limit, Integer start) {
 355         return dao.listUuidAndTitleCacheByAssociatedTaxon(taxonNodeUuids, limit, start);
 356         }
 357
 358     @Override
 359     @Deprecated
 360     public Collection<FieldUnit> listFieldUnitsByAssociatedTaxon(Taxon associatedTaxon, List<OrderHint> orderHints, List<String> propertyPaths) {
 361         return pageRootUnitsByAssociatedTaxon(FieldUnit.class, null, associatedTaxon, null, null, null, null, propertyPaths).getRecords();
 362     }
 363
 364     @Override
 365     public <T extends SpecimenOrObservationBase> Collection<T> listRootUnitsByAssociatedTaxon(Class<T> type, Taxon associatedTaxon, List<OrderHint> orderHints, List<String> propertyPaths) {
 366         return pageRootUnitsByAssociatedTaxon(type, null, associatedTaxon, null, null, null, null, propertyPaths).getRecords();
 367     }
 368
 369     @Override
 370     public <T extends SpecimenOrObservationBase> Pager<T> pageRootUnitsByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 371             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints,
 372             List<String> propertyPaths) {
 373
 374         if (!getSession().contains(associatedTaxon)) {
 375             associatedTaxon = (Taxon) taxonService.load(associatedTaxon.getUuid());
 376         }
 377
 378         // gather the IDs of all relevant root units
 379         Set<UUID> rootUnitUuids = new HashSet<>();
 380         List<SpecimenOrObservationBase> records = listByAssociatedTaxon(null, includeRelationships, associatedTaxon, maxDepth, null, null, orderHints, propertyPaths);
 381         for (SpecimenOrObservationBase<?> specimen : records) {
 382             for (SpecimenOrObservationBase<?> rootUnit : findRootUnits(specimen.getUuid(), null)) {
 383                 if(type == null || type.isAssignableFrom(rootUnit.getClass())) {
 384                     rootUnitUuids.add(rootUnit.getUuid());
 385                 }
 386             }
 387         }
 388         long totalCount = rootUnitUuids.size();
 389         //dao.list() does the paging of the field units. Passing the field units directly to the Pager would not work
 390         List<SpecimenOrObservationBase> rootUnits = dao.list(rootUnitUuids, pageSize, pageNumber, orderHints, propertyPaths);
 391         List<T> castedUnits = new ArrayList<>(rootUnits.size());
 392         for(SpecimenOrObservationBase<?> sob : rootUnits) {
 393             // this cast should be save since the uuids have been filtered by type above
 394             castedUnits.add((T)sob);
 395         }
 396         return new DefaultPagerImpl<>(pageNumber, totalCount, pageSize, castedUnits);
 397     }
 398
 399     @Override
 400     public FieldUnitDTO assembleFieldUnitDTO(FieldUnit fieldUnit) {
 401
 402         if (!getSession().contains(fieldUnit)) {
 403             fieldUnit = (FieldUnit) load(fieldUnit.getUuid());
 404         }
 405         // FIXME the filter for SpecimenOrObservationType.PreservedSpecimen has been preserved from the former implementation (see commit 07e3f63c7d  and older)
 406         // it is questionable if this filter makes sense for all use cases or if it is only a sensible default for the
 407         // compressed specimen table in the cdm-dataportal (see #6816, #6870)
 408         EnumSet<SpecimenOrObservationType> typeIncludeFilter = EnumSet.of(SpecimenOrObservationType.PreservedSpecimen);
 409         FieldUnitDTO fieldUnitDTO = FieldUnitDTO.fromEntity(fieldUnit, null, typeIncludeFilter);
 410         return fieldUnitDTO;
 411     }
 412
 413
 414     @Override
 415     @Transactional
 416     public DerivedUnitDTO assembleDerivedUnitDTO(DerivedUnit derivedUnit) {
 417
 418         if (!getSession().contains(derivedUnit)) {
 419             derivedUnit = (DerivedUnit) load(derivedUnit.getUuid());
 420         }
 421         DerivedUnitDTO derivedUnitDTO = DerivedUnitDTO.fromEntity(derivedUnit, null, null);
 422
 423         // individuals associations
 424         Collection<IndividualsAssociation> individualsAssociations = listIndividualsAssociations(derivedUnit, null, null, null, null);
 425         if(individualsAssociations != null) {
 426             for (IndividualsAssociation individualsAssociation : individualsAssociations) {
 427                 if (individualsAssociation.getInDescription() != null) {
 428                     if (individualsAssociation.getInDescription().isInstanceOf(TaxonDescription.class)) {
 429                         TaxonDescription taxonDescription = HibernateProxyHelper.deproxy(individualsAssociation.getInDescription(), TaxonDescription.class);
 430                         Taxon taxon = taxonDescription.getTaxon();
 431                         if (taxon != null) {
 432                             derivedUnitDTO.addAssociatedTaxon(taxon);
 433                         }
 434                     }
 435                 }
 436             }
 437         }
 438
 439         return derivedUnitDTO;
 440     }
 441
 442     @Override
 443
 444     public String getMostSignificantIdentifier(UUID derivedUnitUuid) {
 445         return dao.findMostSignificantIdentifier(derivedUnitUuid);
 446     }
 447
 448     /**
 449      * TODO there is a very similar function in {@link SpecimenOrObservationBaseDTO#assembleDerivative}.
 450      * If possible we should avoid using this function here by the method in <code>SpecimenOrObservationBaseDTO</code>.
 451      */
 452     private Set<DerivedUnitDTO> getDerivedUnitDTOsFor(SpecimenOrObservationBaseDTO specimenDto, DerivedUnit specimen,
 453             HashMap<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 454
 455         Set<DerivedUnitDTO> derivedUnits = new HashSet<>();
 456         for (DerivationEvent derivationEvent : specimen.getDerivationEvents()) {
 457             for (DerivedUnit derivative : derivationEvent.getDerivatives()) {
 458                 if (!alreadyCollectedSpecimen.containsKey(specimenDto.getUuid())){
 459                     DerivedUnitDTO dto = (DerivedUnitDTO) SpecimenOrObservationDTOFactory.fromEntity(derivative, 0);
 460                     alreadyCollectedSpecimen.put(dto.getUuid(), dto);
 461                     dto.addAllDerivatives(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 462                     derivedUnits.add(dto);
 463                 } else {
 464                     if(alreadyCollectedSpecimen.get(specimenDto.getUuid()).getDerivatives().isEmpty() && !derivative.getDerivationEvents().isEmpty()) {
 465                         // we need to add the missing derivatives!
 466                         SpecimenOrObservationBaseDTO dto = alreadyCollectedSpecimen.get(specimenDto.getUuid());
 467                         alreadyCollectedSpecimen.get(specimenDto.getUuid()).addAllDerivatives(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 468                     }
 469                 }
 470             }
 471         }
 472         return derivedUnits;
 473     }
 474
 475 //    private Set<DerivateDTO> getDerivedUnitDTOsFor(DerivateDTO specimenDto, DerivedUnit specimen, HashMap<UUID, DerivateDTO> alreadyCollectedSpecimen) {
 476 //        Set<DerivateDTO> derivedUnits = new HashSet<>();
 477 ////        load
 478 //        for (DerivationEvent derivationEvent : specimen.getDerivationEvents()) {
 479 //            for (DerivedUnit derivative : derivationEvent.getDerivatives()) {
 480 //                if (!alreadyCollectedSpecimen.containsKey(specimenDto.getUuid())){
 481 //                    PreservedSpecimenDTO dto;
 482 //                    if (derivative instanceof DnaSample){
 483 //                        dto = DNASampleDTO.newInstance(derivative);
 484 //                    }else{
 485 //                        dto = PreservedSpecimenDTO.newInstance(derivative);
 486 //                    }
 487 //                    alreadyCollectedSpecimen.put(dto.getUuid(), dto);
 488 //                    dto.addAllDerivates(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 489 //                    derivedUnits.add(dto);
 490 //                }
 491 //            }
 492 //        }
 493 //        return derivedUnits;
 494 //    }
 495
 496
 497     @SuppressWarnings("unchecked")
 498     @Override
 499     public <T extends SpecimenOrObservationBase> Pager<T> pageByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includedRelationships,
 500             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 501
 502         Set<Taxon> taxa = new HashSet<>();
 503         Set<Integer> occurrenceIds = new HashSet<>();
 504         List<T> occurrences = new ArrayList<>();
 505         boolean includeUnpublished = INCLUDE_UNPUBLISHED;
 506
 507         // Integer limit = PagerUtils.limitFor(pageSize);
 508         // Integer start = PagerUtils.startFor(pageSize, pageNumber);
 509
 510         if (!getSession().contains(associatedTaxon)) {
 511             associatedTaxon = (Taxon) taxonService.load(associatedTaxon.getUuid());
 512         }
 513
 514         if (includedRelationships != null) {
 515             taxa = taxonService.listRelatedTaxa(associatedTaxon, includedRelationships, maxDepth, includeUnpublished, null, null, propertyPaths);
 516         }
 517
 518         taxa.add(associatedTaxon);
 519
 520         for (Taxon taxon : taxa) {
 521             List<T> perTaxonOccurrences = dao.listByAssociatedTaxon(type, taxon, null, null, orderHints, propertyPaths);
 522             for (SpecimenOrObservationBase<?> o : perTaxonOccurrences) {
 523                 occurrenceIds.add(o.getId());
 524             }
 525         }
 526         occurrences = (List<T>) dao.loadList(occurrenceIds, null, propertyPaths);
 527
 528         return new DefaultPagerImpl<>(pageNumber, Long.valueOf(occurrences.size()), pageSize, occurrences);
 529
 530     }
 531
 532     @Override
 533     public <T extends SpecimenOrObservationBase> Pager<T> pageByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 534             String taxonUUID, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 535
 536         UUID uuid = UUID.fromString(taxonUUID);
 537         Taxon taxon = (Taxon) taxonService.load(uuid);
 538         return pageByAssociatedTaxon(type, includeRelationships, taxon, maxDepth, pageSize, pageNumber, orderHints, propertyPaths);
 539
 540     }
 541
 542     @Override
 543     @Transactional
 544     public List<SpecimenOrObservationBaseDTO> listRootUnitDTOsByAssociatedTaxon(Set<TaxonRelationshipEdge> includedRelationships,
 545             UUID associatedTaxonUuid, List<String> propertyPaths) {
 546
 547         Set<Taxon> taxa = new HashSet<>();
 548         Set<SpecimenOrObservationBaseDTO> rootUnitDTOs = new HashSet<>();
 549         boolean includeUnpublished = INCLUDE_UNPUBLISHED;
 550
 551         Taxon associatedTaxon = (Taxon) taxonService.load(associatedTaxonUuid);
 552         if (includedRelationships != null) {
 553             taxa = taxonService.listRelatedTaxa(associatedTaxon, includedRelationships, null, includeUnpublished, null, null, null);
 554         }
 555         taxa.add(associatedTaxon);
 556
 557         HashMap<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedUnits = new HashMap<>();
 558         for (Taxon taxon : taxa) {
 559             // TODO there might be a good potential to speed up the whole processing by collecting all entities first
 560             // and to create the DTOs in a second step
 561             Set<SpecimenOrObservationBase> perTaxonOccurrences = dao.listByAssociatedTaxon(null, taxon, null, null, null, propertyPaths)
 562                     .stream()
 563                     .map(u -> HibernateProxyHelper.deproxy(u, SpecimenOrObservationBase.class))
 564                     .collect(Collectors.toSet());
 565             for (SpecimenOrObservationBase<?> unit : perTaxonOccurrences) {
 566                 unit = HibernateProxyHelper.deproxy(unit);
 567                 if (unit instanceof DerivedUnit){
 568                     DerivedUnitDTO derivativeDTO;
 569                     if (!alreadyCollectedUnits.containsKey(unit.getUuid())){
 570                         DerivedUnit derivedUnit = (DerivedUnit)unit;
 571                         boolean isAssociated = false;
 572                         for (DeterminationEvent determination:derivedUnit.getDeterminations()) {
 573                                 if (determination.getTaxonName() != null && determination.getTaxonName().equals(taxon.getName()) || taxon.equals(determination.getTaxon())){
 574                                         isAssociated = true;
 575                                         break;
 576                                 }
 577                         }
 578                         for (TaxonDescription desc: taxon.getDescriptions()) {
 579                             for (DescriptionElementBase descElement: desc.getElements()) {
 580                                 if (descElement instanceof IndividualsAssociation) {
 581                                     if (((IndividualsAssociation)descElement).getAssociatedSpecimenOrObservation().getUuid().equals(derivedUnit.getUuid())) {
 582                                         isAssociated = true;
 583                                         break;
 584                                     }
 585                                 }
 586                             }
 587                         }
 588
 589                         for (TypeDesignationBase desc: taxon.getName().getTypeDesignations()) {
 590                             if (desc instanceof SpecimenTypeDesignation) {
 591                                 if (((SpecimenTypeDesignation)desc).getTypeSpecimen().getUuid().equals(derivedUnit.getUuid())) {
 592                                     isAssociated = true;
 593                                     break;
 594                                 }
 595                             }
 596
 597                         }
 598
 599                         if (!isAssociated) {
 600                                 continue;
 601                         }
 602                         derivativeDTO = (DerivedUnitDTO) SpecimenOrObservationDTOFactory.fromEntity(derivedUnit, null);
 603                         alreadyCollectedUnits.put(derivativeDTO.getUuid(), derivativeDTO);
 604                         //derivativeDTO.addAllDerivatives(getDerivedUnitDTOsFor(derivativeDTO, derivedUnit, alreadyCollectedUnits));
 605                     }
 606                     derivativeDTO = (DerivedUnitDTO) alreadyCollectedUnits.get(unit.getUuid());
 607                     rootUnitDTOs.addAll(findRootUnitDTOs(derivativeDTO, alreadyCollectedUnits));
 608                 } else {
 609                     // only other option is FieldUnit
 610                     rootUnitDTOs.add(FieldUnitDTO.fromEntity((FieldUnit)unit, 0, null));
 611                 }
 612             }
 613         }
 614
 615         List<SpecimenOrObservationBaseDTO> orderdDTOs = new ArrayList<>(rootUnitDTOs);
 616         // TODO order dtos by date can only be done by string comparison
 617         // the FieldUnitDTO.date needs to be a Partial object for sensible ordering
 618         Collections.sort(orderdDTOs, new Comparator<SpecimenOrObservationBaseDTO>() {
 619
 620             @Override
 621             public int compare(SpecimenOrObservationBaseDTO o1, SpecimenOrObservationBaseDTO o2) {
 622                 if(o1 instanceof FieldUnitDTO && o2 instanceof FieldUnitDTO) {
 623                     FieldUnitDTO fu1 = (FieldUnitDTO)o1;
 624                     FieldUnitDTO fu2 = (FieldUnitDTO)o2;
 625                     //TODO if we want null values and values with missing year as smallest we should use
 626                     //PartialComparator.INSTANCE_NULL_SMALLEST() here
 627                     return PartialComparator.INSTANCE().compare(fu1.getDate(), fu2.getDate());
 628                 }
 629                 if(o1 instanceof DerivedUnitDTO && o2 instanceof DerivedUnitDTO) {
 630                     SpecimenOrObservationBaseDTO du1 = o1;
 631                     SpecimenOrObservationBaseDTO du2 = o2;
 632                     return StringUtils.compare(du1.getLabel(), du2.getLabel());
 633                  }
 634                 if(o1 instanceof FieldUnitDTO && o2 instanceof DerivedUnitDTO) {
 635                     return -1;
 636                 } else {
 637                     return 1;
 638                 }
 639             }
 640
 641         });
 642
 643         return orderdDTOs;
 644     }
 645
 646     @Override
 647     @Transactional
 648     @Deprecated
 649     public  SpecimenOrObservationBaseDTO findByAccessionNumber(String accessionNumberString, List<OrderHint> orderHints)  {
 650         return findByAccessionNumber(accessionNumberString, orderHints);
 651     }
 652
 653     @Override
 654     @Transactional
 655     public  SpecimenOrObservationBaseDTO findByGeneticAccessionNumber(String accessionNumberString, List<OrderHint> orderHints)  {
 656
 657         DnaSample dnaSample = dao.findByGeneticAccessionNumber(accessionNumberString, null);
 658         DerivedUnitDTO dnaSampleDTO;
 659         if (dnaSample != null){
 660             dnaSampleDTO = new DNASampleDTO(dnaSample);
 661             Collection<SpecimenOrObservationBaseDTO> fieldUnitDTOs = this.findRootUnitDTOs(dnaSampleDTO, new HashMap<>());
 662             // FIXME change return type to Collection<FieldUnitDTO>
 663             if(fieldUnitDTOs.isEmpty()) {
 664                 return null;
 665             } else {
 666                return fieldUnitDTOs.iterator().next();
 667             }
 668         }
 669         return null;
 670     }
 671
 672     @Override
 673     public Pager<SearchResult<SpecimenOrObservationBase>> findByFullText(
 674             Class<? extends SpecimenOrObservationBase> clazz, String queryString, RectangleDTO boundingBox, List<Language> languages,
 675             boolean highlightFragments, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints,
 676             List<String> propertyPaths) throws IOException, LuceneParseException {
 677
 678         LuceneSearch luceneSearch = prepareByFullTextSearch(clazz, queryString, boundingBox, languages, highlightFragments);
 679
 680         // --- execute search
 681         TopGroups<BytesRef> topDocsResultSet;
 682         try {
 683             topDocsResultSet = luceneSearch.executeSearch(pageSize, pageNumber);
 684         } catch (ParseException e) {
 685             LuceneParseException parseException = new LuceneParseException(e.getMessage());
 686             parseException.setStackTrace(e.getStackTrace());
 687             throw parseException;
 688         }
 689
 690         Map<CdmBaseType, String> idFieldMap = new HashMap<>();
 691         idFieldMap.put(CdmBaseType.SPECIMEN_OR_OBSERVATIONBASE, "id");
 692
 693         // --- initialize taxa, highlight matches ....
 694         ISearchResultBuilder searchResultBuilder = new SearchResultBuilder(luceneSearch, luceneSearch.getQuery());
 695         @SuppressWarnings("rawtypes")
 696         List<SearchResult<SpecimenOrObservationBase>> searchResults = searchResultBuilder.createResultSet(
 697                 topDocsResultSet, luceneSearch.getHighlightFields(), dao, idFieldMap, propertyPaths);
 698
 699         int totalHits = topDocsResultSet != null ? topDocsResultSet.totalGroupCount : 0;
 700
 701         return new DefaultPagerImpl<>(pageNumber, Long.valueOf(totalHits), pageSize, searchResults);
 702     }
 703
 704     private LuceneSearch prepareByFullTextSearch(Class<? extends SpecimenOrObservationBase> clazz, String queryString, RectangleDTO bbox,
 705             List<Language> languages, boolean highlightFragments) {
 706
 707         Builder finalQueryBuilder = new Builder();
 708         Builder textQueryBuilder = new Builder();
 709
 710         LuceneSearch luceneSearch = new LuceneSearch(luceneIndexToolProvider, FieldUnit.class);
 711         QueryFactory queryFactory = luceneIndexToolProvider.newQueryFactoryFor(FieldUnit.class);
 712
 713         // --- criteria
 714         luceneSearch.setCdmTypRestriction(clazz);
 715         if (queryString != null) {
 716             textQueryBuilder.add(queryFactory.newTermQuery("titleCache", queryString), Occur.SHOULD);
 717             finalQueryBuilder.add(textQueryBuilder.build(), Occur.MUST);
 718         }
 719
 720         // --- spacial query
 721         if (bbox != null) {
 722             finalQueryBuilder.add(QueryFactory.buildSpatialQueryByRange(bbox, "gatheringEvent.exactLocation.point"), Occur.MUST);
 723         }
 724
 725         luceneSearch.setQuery(finalQueryBuilder.build());
 726
 727         // --- sorting
 728         SortField[] sortFields = new SortField[] { SortField.FIELD_SCORE, new SortField("titleCache__sort", SortField.Type.STRING, false) };
 729         luceneSearch.setSortFields(sortFields);
 730
 731         if (highlightFragments) {
 732             luceneSearch.setHighlightFields(queryFactory.getTextFieldNamesAsArray());
 733         }
 734         return luceneSearch;
 735     }
 736
 737
 738     @Override
 739     @Transactional(readOnly=true)
 740     public Collection<FieldUnit> findFieldUnits(UUID derivedUnitUuid, List<String> propertyPaths) {
 741         //It will search recursively over all {@link DerivationEvent}s and get the "originals" ({@link SpecimenOrObservationBase})
 742         //from which this DerivedUnit was derived until all FieldUnits are found.
 743
 744         // FIXME: use HQL queries to avoid entity instantiation and to increase performance
 745
 746         SpecimenOrObservationBase<?> specimen = load(derivedUnitUuid);
 747         Collection<FieldUnit> fieldUnits = new ArrayList<>();
 748         if (specimen == null){
 749             return null;
 750         }
 751         if (specimen.isInstanceOf(FieldUnit.class)) {
 752             fieldUnits.add(HibernateProxyHelper.deproxy(specimen, FieldUnit.class));
 753         }
 754         else if(specimen.isInstanceOf(DerivedUnit.class)){
 755             fieldUnits.addAll(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class).collectRootUnits(FieldUnit.class));
 756         }
 757
 758         fieldUnits = beanInitializer.initializeAll(fieldUnits, propertyPaths);
 759         return fieldUnits;
 760     }
 761
 762     @Override
 763     @Transactional(readOnly=true)
 764     public Collection<SpecimenOrObservationBase> findRootUnits(UUID derivedUnitUuid, List<String> propertyPaths) {
 765
 766         // FIXME: use HQL queries to avoid entity instantiation and to increase performance
 767
 768         SpecimenOrObservationBase<?> specimen = load(derivedUnitUuid);
 769         Collection<SpecimenOrObservationBase> rootUnits = new ArrayList<>();
 770         if (specimen == null){
 771             return null;
 772         }
 773         if (specimen.isInstanceOf(FieldUnit.class)) {
 774             rootUnits.add(HibernateProxyHelper.deproxy(specimen, FieldUnit.class));
 775         }
 776         else if(specimen.isInstanceOf(DerivedUnit.class)){
 777             rootUnits.addAll(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class).collectRootUnits(SpecimenOrObservationBase.class));
 778         }
 779
 780         rootUnits = beanInitializer.initializeAll(rootUnits, propertyPaths);
 781         return rootUnits;
 782     }
 783
 784     @Override
 785     public Collection<SpecimenOrObservationBaseDTO> findRootUnitDTOs(UUID unitUUID) {
 786
 787         SpecimenOrObservationBase<?> entity = load(unitUUID);
 788         SpecimenOrObservationBaseDTO derivedUnitDTO = SpecimenOrObservationDTOFactory.fromEntity(entity);
 789         Collection<SpecimenOrObservationBaseDTO> rootUnitDTOs = new ArrayList<>();
 790         if(derivedUnitDTO != null) {
 791             if(derivedUnitDTO instanceof FieldUnitDTO) {
 792                 rootUnitDTOs.add(derivedUnitDTO);
 793             } else {
 794                 Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen = new HashMap<>();
 795                 rootUnitDTOs  = findRootUnitDTOs((DerivedUnitDTO)derivedUnitDTO, alreadyCollectedSpecimen);
 796             }
 797         }
 798
 799         return rootUnitDTOs;
 800
 801     }
 802
 803     /**
 804      * Recursively searches all {@link DerivationEvent}s to find all "originals" ({@link SpecimenOrObservationBase})
 805      * from which this DerivedUnit was derived until all FieldUnits are found.
 806      * <p>
 807      * <b>NOTE:</b> The recursive search still is a bit incomplete and may miss originals in the rare case where a
 808      * derivative has more than one original. (see https://dev.e-taxonomy.eu/redmine/issues/9253)
 809      *
 810      * @param derivedUnitDTO
 811      *  The DerivedUnitDTO to start the search from.
 812      * @param alreadyCollectedSpecimen
 813      *  A map to hold all originals that have been sees during the recursive walk.
 814      * @return
 815      *  The collection of all Field Units that are accessible from the derivative from where the search was started.
 816      */
 817     public Collection<SpecimenOrObservationBaseDTO> findRootUnitDTOs(DerivedUnitDTO derivedUnitDTO,
 818             Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 819
 820         HashMap<UUID, SpecimenOrObservationBaseDTO> rootUnitDTOs = new HashMap<>();
 821         _findRootUnitDTOs(derivedUnitDTO, rootUnitDTOs, alreadyCollectedSpecimen);
 822         return rootUnitDTOs.values();
 823
 824     }
 825
 826     /**
 827      * Method for recursive calls, must only be used by {@link #findRootUnitDTOs(DerivedUnitDTO, HashMap)}
 828      * <p>
 829      * It will search recursively over all {@link DerivationEvent}s and get the "originals" ({@link SpecimenOrObservationBase})
 830      * from which this DerivedUnit was derived until all FieldUnits are found.
 831      */
 832     private void _findRootUnitDTOs(DerivedUnitDTO derivedUnitDTO, Map<UUID, SpecimenOrObservationBaseDTO> rootUnitDTOs,
 833                 Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 834
 835         List<String> propertyPaths = new ArrayList<>();
 836
 837         // add the supplied DTO the the alreadyCollectedSpecimen if not yet there
 838         if(!alreadyCollectedSpecimen.containsKey(derivedUnitDTO.getUuid())) {
 839             alreadyCollectedSpecimen.put(derivedUnitDTO.getUuid(), derivedUnitDTO);
 840         }
 841
 842         List<SpecimenOrObservationBase> originals = dao.findOriginalsForDerivedUnit(derivedUnitDTO.getUuid(), propertyPaths);
 843         if (originals.size() > 0){
 844             if (originals.size() > 1){
 845                 logger.warn("The derived unit with uuid " + derivedUnitDTO.getUuid() + "has more than one orginal");
 846             }
 847             // FIXME allow handling multiple originals
 848             SpecimenOrObservationBase<?> original = originals.get(0);
 849             original = HibernateProxyHelper.deproxy(original);
 850
 851             if (alreadyCollectedSpecimen.containsKey(original.getUuid())){
 852                 alreadyCollectedSpecimen.get(original.getUuid()).addDerivative(derivedUnitDTO);
 853     //            if ( alreadyCollectedSpecimen.get(specimen.getUuid()) instanceof FieldUnitDTO){
 854     //                ((FieldUnitDTO)alreadyCollectedSpecimen.get(specimen.getUuid())).getTaxonRelatedDerivedUnits().add(derivedUnitDTO.getUuid());
 855     //            }
 856             }else{
 857                 if(!rootUnitDTOs.containsKey(original.getUuid())){
 858                     // the direct derivatives of the field unit are added in the factory method, so it is guaranteed that
 859                     // the derivedUnitDTO is already contained.
 860                     // ----
 861                     // Don't assemble derivatives for originals since we have them collected already
 862                     // when ascending to the originals, we only want to collect those derivatives which are on the path up to the root
 863                     final Integer maxDepth = 0;
 864                     SpecimenOrObservationBaseDTO originalDTO = SpecimenOrObservationDTOFactory.fromEntity(original, maxDepth);
 865                     originalDTO.addDerivative(derivedUnitDTO);
 866                     alreadyCollectedSpecimen.put(originalDTO.getUuid(), originalDTO);
 867                     if (original instanceof FieldUnit){
 868                         rootUnitDTOs.put(originalDTO.getUuid(), originalDTO);
 869                     }else{
 870                         _findRootUnitDTOs((DerivedUnitDTO) originalDTO, rootUnitDTOs, alreadyCollectedSpecimen);
 871                     }
 872                 } else {
 873                     SpecimenOrObservationBaseDTO previouslyFoundRootUnit = rootUnitDTOs.get(original.getUuid());
 874                     if(!previouslyFoundRootUnit.getDerivatives().stream().anyMatch(uDTO -> uDTO.getUuid().equals(derivedUnitDTO.getUuid()))) {
 875                         previouslyFoundRootUnit.addDerivative(derivedUnitDTO);
 876                     }
 877                 }
 878             }
 879         } else {
 880             rootUnitDTOs.put(derivedUnitDTO.getUuid(), derivedUnitDTO);
 881         }
 882
 883     }
 884
 885     @Override
 886     @Transactional(readOnly=true)
 887     public FieldUnitDTO loadFieldUnitDTO(UUID derivedUnitUuid) {
 888
 889         FieldUnitDTO fieldUnitDTO = null;
 890         DerivedUnitDTO derivedUnitDTO = null;
 891
 892         Map<UUID, SpecimenOrObservationBaseDTO> cycleDetectionMap = new HashMap<>();
 893         SpecimenOrObservationBase<?> derivative = dao.load(derivedUnitUuid);
 894         if(derivative != null){
 895             if (derivative instanceof FieldUnit){
 896                 fieldUnitDTO = FieldUnitDTO.fromEntity((FieldUnit)derivative);
 897                 return fieldUnitDTO;
 898             } else {
 899                 // must be a DerivedUnit otherwise
 900                 derivedUnitDTO = DerivedUnitDTO.fromEntity((DerivedUnit)derivative);
 901                 while(true){
 902
 903                     Set<SpecimenOrObservationBaseDTO> originals = originalDTOs(derivedUnitDTO.getUuid());
 904
 905                     if(originals.isEmpty()){
 906                         break;
 907                     }
 908                     if (originals.size() > 1){
 909                         logger.debug("The derived unit with uuid " + derivedUnitUuid + "has more than one orginal, ignoring all but the first one.");
 910                     }
 911
 912                     SpecimenOrObservationBaseDTO originalDTO = originals.iterator().next();
 913
 914                     // cycle detection and handling
 915                     if(cycleDetectionMap.containsKey(originalDTO.getUuid())){
 916                         // cycle detected!!!
 917                         try {
 918                             throw new Exception();
 919                         } catch(Exception e){
 920                             logger.error("Cycle in derivate graph detected at DerivedUnit with uuid=" + originalDTO.getUuid() , e);
 921                         }
 922                         // to solve the situation for the output we remove the derivate from the more distant graph node
 923                         // by removing it from the derivatives of its original
 924                         // but let the derivate to be added to the original which is closer to the FieldUnit (below at originalDTO.addDerivate(derivedUnitDTO);)
 925                         for(SpecimenOrObservationBaseDTO seenOriginal: cycleDetectionMap.values()){
 926                             for(SpecimenOrObservationBaseDTO derivateDTO : seenOriginal.getDerivatives()){
 927                                 if(derivateDTO.equals(originalDTO)){
 928                                     seenOriginal.getDerivatives().remove(originalDTO);
 929                                 }
 930                             }
 931                         }
 932                     } else {
 933                         cycleDetectionMap.put(originalDTO.getUuid(), originalDTO);
 934                     }
 935
 936                     if (originalDTO instanceof FieldUnitDTO){
 937                         fieldUnitDTO = (FieldUnitDTO)originalDTO;
 938                         break;
 939                     }else{
 940                         // So this must be a DerivedUnitDTO
 941                         if (derivedUnitDTO == null){
 942                             derivedUnitDTO = (DerivedUnitDTO)originalDTO;
 943                         } else {
 944                             derivedUnitDTO = (DerivedUnitDTO)originalDTO;
 945                         }
 946                     }
 947                 }
 948             }
 949         }
 950         return fieldUnitDTO;
 951
 952     }
 953
 954     /**
 955      * @param originalDTO
 956      * @return
 957      */
 958     private Set<SpecimenOrObservationBaseDTO> originalDTOs(UUID derivativeUuid) {
 959
 960         Set<SpecimenOrObservationBaseDTO> dtos = new HashSet<>();
 961         List<SpecimenOrObservationBase> specimens = dao.findOriginalsForDerivedUnit(derivativeUuid, null);
 962         for(SpecimenOrObservationBase sob : specimens){
 963             if(sob instanceof FieldUnit) {
 964                 dtos.add(FieldUnitDTO.fromEntity((FieldUnit)sob));
 965             } else {
 966                 dtos.add(DerivedUnitDTO.fromEntity((DerivedUnit)sob));
 967             }
 968         }
 969         return dtos;
 970     }
 971
 972
 973     @Override
 974     @Transactional(readOnly = false)
 975     public UpdateResult moveSequence(DnaSample from, DnaSample to, Sequence sequence) {
 976         return moveSequence(from.getUuid(), to.getUuid(), sequence.getUuid());
 977     }
 978
 979     @Override
 980     @Transactional(readOnly = false)
 981     public UpdateResult moveSequence(UUID fromUuid, UUID toUuid, UUID sequenceUuid) {
 982         // reload specimens to avoid session conflicts
 983         DnaSample from = (DnaSample) load(fromUuid);
 984         DnaSample to = (DnaSample) load(toUuid);
 985         Sequence sequence = sequenceService.load(sequenceUuid);
 986
 987         if (from == null || to == null || sequence == null) {
 988             throw new TransientObjectException("One of the CDM entities has not been saved to the data base yet. Moving only works for persisted/saved CDM entities.\n" +
 989                     "Operation was move "+sequence+ " from "+from+" to "+to);
 990         }
 991         UpdateResult result = new UpdateResult();
 992         from.removeSequence(sequence);
 993         saveOrUpdate(from);
 994         to.addSequence(sequence);
 995         saveOrUpdate(to);
 996         result.setStatus(Status.OK);
 997         result.addUpdatedObject(from);
 998         result.addUpdatedObject(to);
 999         return result;
1000     }
1001
1002     @Override
1003     @Transactional(readOnly = false)
1004     public boolean moveDerivate(SpecimenOrObservationBase<?> from, SpecimenOrObservationBase<?> to, DerivedUnit derivate) {
1005         return moveDerivate(from!=null?from.getUuid():null, to.getUuid(), derivate.getUuid()).isOk();
1006     }
1007
1008     @Override
1009     @Transactional(readOnly = false)
1010     public UpdateResult moveDerivate(UUID specimenFromUuid, UUID specimenToUuid, UUID derivateUuid) {
1011         // reload specimens to avoid session conflicts
1012         SpecimenOrObservationBase<?> from = null;
1013         if(specimenFromUuid!=null){
1014             from = load(specimenFromUuid);
1015         }
1016         SpecimenOrObservationBase<?> to = load(specimenToUuid);
1017         DerivedUnit derivate = (DerivedUnit) load(derivateUuid);
1018
1019         if ((specimenFromUuid!=null && from == null) || to == null || derivate == null) {
1020             throw new TransientObjectException("One of the CDM entities has not been saved to the data base yet. Moving only works for persisted/saved CDM entities.\n" +
1021                     "Operation was move "+derivate+ " from "+from+" to "+to);
1022         }
1023         UpdateResult result = new UpdateResult();
1024         SpecimenOrObservationType derivateType = derivate.getRecordBasis();
1025         SpecimenOrObservationType toType = to.getRecordBasis();
1026         // check if type is a sub derivate type
1027         if(toType==SpecimenOrObservationType.FieldUnit //moving to FieldUnit always works
1028                 || derivateType==SpecimenOrObservationType.Media //moving media always works
1029                 || (derivateType.isKindOf(toType) && toType!=derivateType)){ //moving only to parent derivate type
1030             if(from!=null){
1031                 // remove derivation event from parent specimen of dragged object
1032                 DerivationEvent eventToRemove = null;
1033                 for (DerivationEvent event : from.getDerivationEvents()) {
1034                     if (event.getDerivatives().contains(derivate)) {
1035                         eventToRemove = event;
1036                         break;
1037                     }
1038                 }
1039                 from.removeDerivationEvent(eventToRemove);
1040                 if(eventToRemove!=null){
1041                     // add new derivation event to target and copy the event parameters of the old one
1042                     DerivationEvent derivedFromNewOriginalEvent = DerivationEvent.NewSimpleInstance(to, derivate, null);
1043                     derivedFromNewOriginalEvent.setActor(eventToRemove.getActor());
1044                     derivedFromNewOriginalEvent.setDescription(eventToRemove.getDescription());
1045                     derivedFromNewOriginalEvent.setInstitution(eventToRemove.getInstitution());
1046                     derivedFromNewOriginalEvent.setTimeperiod(eventToRemove.getTimeperiod());
1047                     derivedFromNewOriginalEvent.setType(eventToRemove.getType());
1048                     to.addDerivationEvent(derivedFromNewOriginalEvent);
1049                     derivate.setDerivedFrom(derivedFromNewOriginalEvent);
1050                 }
1051             }
1052             else{
1053                 //derivative had no parent before so we use empty derivation event
1054                 DerivationEvent derivedFromNewOriginalEvent = DerivationEvent.NewSimpleInstance(to, derivate, null);
1055                 to.addDerivationEvent(derivedFromNewOriginalEvent);
1056                 derivate.setDerivedFrom(derivedFromNewOriginalEvent);
1057             }
1058
1059             if(from!=null){
1060                 saveOrUpdate(from);
1061             }
1062             saveOrUpdate(to);
1063             result.setStatus(Status.OK);
1064             result.addUpdatedObject(from);
1065             result.addUpdatedObject(to);
1066         } else {
1067             result.setStatus(Status.ERROR);
1068         }
1069         return result;
1070     }
1071
1072     @Override
1073     public DeleteResult isDeletable(UUID specimenUuid, DeleteConfiguratorBase config) {
1074         DeleteResult deleteResult = new DeleteResult();
1075         SpecimenOrObservationBase<?> specimen = this.load(specimenUuid);
1076         SpecimenDeleteConfigurator specimenDeleteConfigurator = (SpecimenDeleteConfigurator) config;
1077
1078         // check elements found by super method
1079         Set<CdmBase> relatedObjects = super.isDeletable(specimenUuid, config).getRelatedObjects();
1080         for (CdmBase cdmBase : relatedObjects) {
1081             // check for type designation
1082             if (cdmBase.isInstanceOf(SpecimenTypeDesignation.class) && !specimenDeleteConfigurator.isDeleteFromTypeDesignation()) {
1083                 deleteResult.setAbort();
1084                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is a type specimen."));
1085                 deleteResult.addRelatedObject(cdmBase);
1086                 break;
1087             }
1088             // check for IndividualsAssociations
1089             else if (cdmBase.isInstanceOf(IndividualsAssociation.class) && !specimenDeleteConfigurator.isDeleteFromIndividualsAssociation()) {
1090                 deleteResult.setAbort();
1091                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is still associated via IndividualsAssociations"));
1092                 deleteResult.addRelatedObject(cdmBase);
1093                 break;
1094             }
1095             // check for taxon description
1096             else if(cdmBase.isInstanceOf(TaxonDescription.class)
1097                     && HibernateProxyHelper.deproxy(cdmBase, TaxonDescription.class).getDescribedSpecimenOrObservation().equals(specimen)
1098                     && !specimenDeleteConfigurator.isDeleteFromDescription()){
1099                 deleteResult.setAbort();
1100                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is still used as \"Described Specimen\" in a taxon description."));
1101                 deleteResult.addRelatedObject(cdmBase);
1102                 break;
1103             }
1104             // check for children and parents (derivation events)
1105             else if (cdmBase.isInstanceOf(DerivationEvent.class)) {
1106                 DerivationEvent derivationEvent = HibernateProxyHelper.deproxy(cdmBase, DerivationEvent.class);
1107                 // check if derivation event is empty
1108                 if (!derivationEvent.getDerivatives().isEmpty() && derivationEvent.getOriginals().contains(specimen)) {
1109                     // if derivationEvent is the childEvent and contains derivations
1110 //                    if (derivationEvent.getDerivatives().contains(specimen)) {
1111 //                        //if it is the parent event the specimen is still deletable
1112 //                        continue;
1113 //                    }
1114                     if(!specimenDeleteConfigurator.isDeleteChildren()){
1115                         //if children should not be deleted then it is undeletable
1116                         deleteResult.setAbort();
1117                         deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation still has child derivatives."));
1118                         deleteResult.addRelatedObject(cdmBase);
1119                         break;
1120                     }
1121                     else{
1122                         // check all children if they can be deleted
1123                         Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1124                         DeleteResult childResult = new DeleteResult();
1125                         for (DerivedUnit derivedUnit : derivatives) {
1126                             childResult.includeResult(isDeletable(derivedUnit.getUuid(), specimenDeleteConfigurator));
1127                         }
1128                         if (!childResult.isOk()) {
1129                             deleteResult.setAbort();
1130                             deleteResult.includeResult(childResult);
1131                             deleteResult.addRelatedObject(cdmBase);
1132                             break;
1133                         }
1134                     }
1135                 }
1136             }
1137             // check for amplification
1138             else if (cdmBase.isInstanceOf(AmplificationResult.class)
1139                     && !specimenDeleteConfigurator.isDeleteMolecularData()
1140                     && !specimenDeleteConfigurator.isDeleteChildren()) {
1141                 deleteResult.setAbort();
1142                 deleteResult.addException(new ReferencedObjectUndeletableException("DnaSample is used in amplification results."));
1143                 deleteResult.addRelatedObject(cdmBase);
1144                 break;
1145             }
1146             // check for sequence
1147             else if (cdmBase.isInstanceOf(Sequence.class)
1148                     && !specimenDeleteConfigurator.isDeleteMolecularData()
1149                     && !specimenDeleteConfigurator.isDeleteChildren()) {
1150                 deleteResult.setAbort();
1151                 deleteResult.addException(new ReferencedObjectUndeletableException("DnaSample is used in sequences."));
1152                 deleteResult.addRelatedObject(cdmBase);
1153                 break;
1154             }
1155         }
1156         if (deleteResult.isOk()) {
1157             //add all related object if deletion is OK so they can be handled by the delete() method
1158             deleteResult.addRelatedObjects(relatedObjects);
1159         }
1160         return deleteResult;
1161     }
1162
1163     @Transactional(readOnly = false)
1164     @Override
1165     public DeleteResult delete(UUID specimenUuid, SpecimenDeleteConfigurator config) {
1166         return delete(load(specimenUuid), config);
1167     }
1168
1169     @Transactional(readOnly = false)
1170     @Override
1171     public DeleteResult delete(SpecimenOrObservationBase<?> specimen, SpecimenDeleteConfigurator config) {
1172         specimen = HibernateProxyHelper.deproxy(specimen, SpecimenOrObservationBase.class);
1173
1174         DeleteResult deleteResult = isDeletable(specimen.getUuid(), config);
1175         if (!deleteResult.isOk()) {
1176             return deleteResult;
1177         }
1178
1179         if (config.isDeleteChildren()) {
1180             Set<DerivationEvent> derivationEvents = specimen.getDerivationEvents();
1181             //clone to avoid concurrent modification
1182             //can happen if the child is deleted and deleted its own derivedFrom event
1183             Set<DerivationEvent> derivationEventsClone = new HashSet<>(derivationEvents);
1184             for (DerivationEvent derivationEvent : derivationEventsClone) {
1185                 Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1186                 Iterator<DerivedUnit> it = derivatives.iterator();
1187                 Set<DerivedUnit> derivativesToDelete = new HashSet<>();
1188                 while (it.hasNext()) {
1189                     DerivedUnit unit = it.next();
1190                     derivativesToDelete.add(unit);
1191                 }
1192                 for (DerivedUnit unit: derivativesToDelete){
1193                     DeleteResult derivedDeleteResult = delete(unit, config);
1194                     deleteResult.includeResult(derivedDeleteResult);
1195                 }
1196             }
1197         }
1198
1199         // check related objects
1200         Set<CdmBase> relatedObjects = deleteResult.getRelatedObjects();
1201
1202         for (CdmBase relatedObject : relatedObjects) {
1203             // check for TypeDesignations
1204             if (relatedObject.isInstanceOf(SpecimenTypeDesignation.class)) {
1205                 SpecimenTypeDesignation designation = HibernateProxyHelper.deproxy(relatedObject, SpecimenTypeDesignation.class);
1206                 designation.setTypeSpecimen(null);
1207                 List<TaxonName> typifiedNames = new ArrayList<>();
1208                 typifiedNames.addAll(designation.getTypifiedNames());
1209                 for (TaxonName taxonName : typifiedNames) {
1210                     taxonName.removeTypeDesignation(designation);
1211                 }
1212             }
1213             // delete IndividualsAssociation
1214             if (relatedObject.isInstanceOf(IndividualsAssociation.class)) {
1215                 IndividualsAssociation association = HibernateProxyHelper.deproxy(relatedObject, IndividualsAssociation.class);
1216                 association.setAssociatedSpecimenOrObservation(null);
1217                 association.getInDescription().removeElement(association);
1218             }
1219             // check for "described specimen" (deprecated)
1220             if (relatedObject.isInstanceOf(TaxonDescription.class)) {
1221                 TaxonDescription description = HibernateProxyHelper.deproxy(relatedObject, TaxonDescription.class);
1222                 description.setDescribedSpecimenOrObservation(null);
1223             }
1224             // check for specimen description
1225             if (relatedObject.isInstanceOf(SpecimenDescription.class)) {
1226                 SpecimenDescription specimenDescription = HibernateProxyHelper.deproxy(relatedObject, SpecimenDescription.class);
1227                 specimenDescription.setDescribedSpecimenOrObservation(null);
1228                 // check if description is a description of the given specimen
1229                 if (specimen.getDescriptions().contains(specimenDescription)) {
1230                     specimen.removeDescription(specimenDescription);
1231                 }
1232                 DeleteResult descriptionDelete = descriptionService.isDeletable(specimenDescription.getUuid(), null);
1233                 if (descriptionDelete.isOk()){
1234                     deleteResult.includeResult(descriptionService.delete(specimenDescription));
1235                 }
1236             }
1237             // check for amplification
1238             if (relatedObject.isInstanceOf(AmplificationResult.class)) {
1239                 AmplificationResult amplificationResult = HibernateProxyHelper.deproxy(relatedObject, AmplificationResult.class);
1240                 amplificationResult.getDnaSample().removeAmplificationResult(amplificationResult);
1241             }
1242             // check for sequence
1243             if (relatedObject.isInstanceOf(Sequence.class)) {
1244                 Sequence sequence = HibernateProxyHelper.deproxy(relatedObject, Sequence.class);
1245                 sequence.getDnaSample().removeSequence(sequence);
1246             }
1247             // check for children and parents (derivation events)
1248             if (relatedObject.isInstanceOf(DerivationEvent.class)) {
1249                 DerivationEvent derivationEvent = HibernateProxyHelper.deproxy(relatedObject, DerivationEvent.class);
1250                 // parent derivation event (derivedFrom)
1251                 if (derivationEvent.getDerivatives().contains(specimen) && specimen.isInstanceOf(DerivedUnit.class)) {
1252                     derivationEvent.removeDerivative(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class));
1253                     if (derivationEvent.getDerivatives().isEmpty()) {
1254                         Set<SpecimenOrObservationBase> originals = new HashSet<>();
1255                         originals.addAll(derivationEvent.getOriginals());
1256                         for (SpecimenOrObservationBase specimenOrObservationBase : originals) {
1257                             specimenOrObservationBase.removeDerivationEvent(derivationEvent);
1258                             deleteResult.addUpdatedObject(specimenOrObservationBase);
1259                         }
1260                         // if derivationEvent has no derivates anymore, delete it
1261                         deleteResult.includeResult(eventService.delete(derivationEvent));
1262                     }
1263                 }
1264                 else{
1265                     //child derivation events should not occur since we delete the hierarchy from bottom to top
1266                 }
1267             }
1268         }
1269         if (specimen instanceof FieldUnit){
1270             FieldUnit fieldUnit = HibernateProxyHelper.deproxy(specimen, FieldUnit.class);
1271             GatheringEvent event = fieldUnit.getGatheringEvent();
1272             fieldUnit.setGatheringEvent(null);
1273             if (event != null){
1274                 DeleteResult result = eventService.isDeletable(event.getUuid(), null);
1275                 if (result.isOk()){
1276                     deleteResult.includeResult( eventService.delete(event));
1277                 }
1278             }
1279
1280         }
1281         deleteResult.includeResult(delete(specimen));
1282
1283         return deleteResult;
1284     }
1285
1286     @Override
1287     public Collection<IndividualsAssociation> listIndividualsAssociations(SpecimenOrObservationBase<?> specimen, Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1288         return dao.listIndividualsAssociations(specimen, limit, start, orderHints, propertyPaths);
1289     }
1290
1291     /**
1292      * {@inheritDoc}
1293      */
1294     @Override
1295     public Collection<TaxonBase<?>> listAssociatedTaxa(SpecimenOrObservationBase<?> specimen, Integer limit,
1296             Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1297         return listAssociatedTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1298     }
1299     @Override
1300     public Collection<TaxonBase<?>> listAssociatedTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished,
1301             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1302         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1303
1304         //individuals associations
1305         associatedTaxa.addAll(listIndividualsAssociationTaxa(specimen, includeUnpublished, limit, start, orderHints, propertyPaths));
1306         //type designation
1307         if(specimen.isInstanceOf(DerivedUnit.class)){
1308             associatedTaxa.addAll(listTypeDesignationTaxa(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class),
1309                   includeUnpublished, limit, start, orderHints, propertyPaths));
1310         }
1311         //determinations
1312         associatedTaxa.addAll(listDeterminedTaxa(specimen, includeUnpublished, limit, start, orderHints, propertyPaths));
1313
1314         return associatedTaxa;
1315     }
1316
1317
1318
1319     /**
1320      * {@inheritDoc}
1321      */
1322     @Override
1323     public Collection<TaxonBase<?>> listDeterminedTaxa(SpecimenOrObservationBase<?> specimen, Integer limit,
1324             Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1325         return listDeterminedTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1326     }
1327     @Override
1328     public Collection<TaxonBase<?>> listDeterminedTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished, Integer limit, Integer start,
1329             List<OrderHint> orderHints, List<String> propertyPaths) {
1330         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1331         for (DeterminationEvent determinationEvent : listDeterminationEvents(specimen, limit, start, orderHints, propertyPaths)) {
1332             if(determinationEvent.getIdentifiedUnit().equals(specimen)){
1333                 if(determinationEvent.getTaxon()!=null){
1334                     associatedTaxa.add(taxonService.load(determinationEvent.getTaxon().getUuid(), includeUnpublished, propertyPaths));
1335                 }
1336                 if(determinationEvent.getTaxonName()!=null){
1337                     Collection<TaxonBase> taxonBases = determinationEvent.getTaxonName().getTaxonBases();
1338                     for (TaxonBase taxonBase : taxonBases) {
1339                         associatedTaxa.add(taxonService.load(taxonBase.getUuid(), includeUnpublished, propertyPaths));
1340                     }
1341                 }
1342             }
1343         }
1344         return associatedTaxa;
1345     }
1346
1347     /**
1348      * {@inheritDoc}
1349      */
1350     @Override
1351     public Collection<TaxonBase<?>> listTypeDesignationTaxa(DerivedUnit specimen, Integer limit, Integer start,
1352             List<OrderHint> orderHints, List<String> propertyPaths) {
1353         return listTypeDesignationTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1354     }
1355     @Override
1356     public Collection<TaxonBase<?>> listTypeDesignationTaxa(DerivedUnit specimen, boolean includeUnpublished,
1357             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1358         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1359         for (SpecimenTypeDesignation typeDesignation : listTypeDesignations(specimen, limit, start, orderHints, propertyPaths)) {
1360             if(typeDesignation.getTypeSpecimen().equals(specimen)){
1361                 Set<TaxonName> typifiedNames = typeDesignation.getTypifiedNames();
1362                 for (TaxonName taxonName : typifiedNames) {
1363                     Set<Taxon> taxa = taxonName.getTaxa();
1364                     for (Taxon taxon : taxa) {
1365                         associatedTaxa.add(taxonService.load(taxon.getUuid(), includeUnpublished, propertyPaths));
1366                     }
1367                 }
1368             }
1369         }
1370         return associatedTaxa;
1371     }
1372
1373     /**
1374      * {@inheritDoc}
1375      */
1376     @Override
1377     public Collection<TaxonBase<?>> listIndividualsAssociationTaxa(SpecimenOrObservationBase<?> specimen, Integer limit,
1378             Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1379         return listIndividualsAssociationTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1380     }
1381
1382     @Override
1383     public Collection<TaxonBase<?>> listIndividualsAssociationTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished,
1384             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1385         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1386         for (IndividualsAssociation individualsAssociation : listIndividualsAssociations(specimen, null, null, null, null)) {
1387             if(individualsAssociation.getInDescription().isInstanceOf(TaxonDescription.class)){
1388                 TaxonDescription taxonDescription = HibernateProxyHelper.deproxy(individualsAssociation.getInDescription(), TaxonDescription.class);
1389                 if(taxonDescription.getTaxon()!=null){
1390                     associatedTaxa.add(taxonService.load(taxonDescription.getTaxon().getUuid(), includeUnpublished, propertyPaths));
1391                 }
1392             }
1393         }
1394         return associatedTaxa;
1395     }
1396
1397     @Override
1398     public Collection<DeterminationEvent> listDeterminationEvents(SpecimenOrObservationBase<?> specimen,
1399             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1400         return dao.listDeterminationEvents(specimen, limit, start, orderHints, propertyPaths);
1401     }
1402
1403     @Override
1404     public Map<DerivedUnit, Collection<SpecimenTypeDesignation>> listTypeDesignations(
1405             Collection<DerivedUnit> specimens, Integer limit, Integer start,
1406             List<OrderHint> orderHints, List<String> propertyPaths) {
1407         Map<DerivedUnit, Collection<SpecimenTypeDesignation>> typeDesignationMap = new HashMap<>();
1408         for (DerivedUnit specimen : specimens) {
1409             Collection<SpecimenTypeDesignation> typeDesignations = listTypeDesignations(specimen, limit, start, orderHints, propertyPaths);
1410             typeDesignationMap.put(specimen, typeDesignations);
1411         }
1412         return typeDesignationMap;
1413     }
1414
1415     @Override
1416     public Collection<SpecimenTypeDesignation> listTypeDesignations(DerivedUnit specimen,
1417             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1418         return dao.listTypeDesignations(specimen, limit, start, orderHints, propertyPaths);
1419     }
1420
1421     @Override
1422     public Collection<DescriptionBase<?>> listDescriptionsWithDescriptionSpecimen(
1423             SpecimenOrObservationBase<?> specimen, Integer limit, Integer start, List<OrderHint> orderHints,
1424             List<String> propertyPaths) {
1425         return dao.listDescriptionsWithDescriptionSpecimen(specimen, limit, start, orderHints, propertyPaths);
1426     }
1427
1428     @Override
1429     public Collection<DescriptionElementBase> getCharacterDataForSpecimen(UUID specimenUuid) {
1430         SpecimenOrObservationBase<?> specimen = load(specimenUuid);
1431         if (specimen != null) {
1432             return specimen.characterData();
1433         }
1434         else{
1435             throw new DataRetrievalFailureException("Specimen with the given uuid not found in the data base");
1436         }
1437     }
1438
1439     @Override
1440     public long countByTitle(IIdentifiableEntityServiceConfigurator<SpecimenOrObservationBase> config){
1441         if (config instanceof FindOccurrencesConfigurator) {
1442             FindOccurrencesConfigurator occurrenceConfig = (FindOccurrencesConfigurator) config;
1443             Taxon taxon = null;
1444             if(occurrenceConfig.getAssociatedTaxonUuid()!=null){
1445                 TaxonBase<?> taxonBase = taxonService.load(occurrenceConfig.getAssociatedTaxonUuid());
1446                 if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1447                     taxon = HibernateProxyHelper.deproxy(taxonBase, Taxon.class);
1448                 }
1449             }
1450             TaxonName taxonName = null;
1451             if(occurrenceConfig.getAssociatedTaxonNameUuid()!=null){
1452                 taxonName = nameService.load(occurrenceConfig.getAssociatedTaxonNameUuid());
1453             }
1454             /*TODO: #6484 Neither isRetrieveIndirectlyAssociatedSpecimens() nor the AssignmentStatus
1455              * is currently reflected in the HQL query. So using these in the count method will
1456              * significantly slow down this method as we have to retrieve the entities instead of
1457              * just the amount.
1458              */
1459             if(occurrenceConfig.isRetrieveIndirectlyAssociatedSpecimens() || !occurrenceConfig.getAssignmentStatus().equals(AssignmentStatus.ALL_SPECIMENS)){
1460                 List<SpecimenOrObservationBase> occurrences = new ArrayList<>();
1461                 List<SpecimenOrObservationBase> sobs = dao.findOccurrences(occurrenceConfig.getClazz(),
1462                         occurrenceConfig.getTitleSearchStringSqlized(), occurrenceConfig.getSignificantIdentifier(),
1463                         occurrenceConfig.getSpecimenType(), taxon, taxonName, occurrenceConfig.getMatchMode(), null, null,
1464                         occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1465                 occurrences.addAll(sobs);
1466                 occurrences = filterOccurencesByAssignmentAndHierarchy(occurrenceConfig, occurrences, taxon, taxonName);
1467                 return occurrences.size();
1468             }
1469
1470             return dao.countOccurrences(occurrenceConfig.getClazz(),
1471                     occurrenceConfig.getTitleSearchStringSqlized(), occurrenceConfig.getSignificantIdentifier(),
1472                     occurrenceConfig.getSpecimenType(), taxon, taxonName, occurrenceConfig.getMatchMode(), null, null,
1473                     occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1474         }
1475         else{
1476             return super.countByTitle(config);
1477         }
1478     }
1479
1480     @Override
1481     public Pager<UuidAndTitleCache<SpecimenOrObservationBase>> findByTitleUuidAndTitleCache(
1482             FindOccurrencesConfigurator config){
1483         List<UuidAndTitleCache<SpecimenOrObservationBase>> occurrences = new ArrayList<>();
1484         Taxon taxon = null;
1485         if(config.getAssociatedTaxonUuid()!=null){
1486             TaxonBase<?> taxonBase = taxonService.load(config.getAssociatedTaxonUuid());
1487             if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1488                 taxon = CdmBase.deproxy(taxonBase, Taxon.class);
1489             }
1490         }
1491         TaxonName taxonName = null;
1492         if(config.getAssociatedTaxonNameUuid()!=null){
1493             taxonName = nameService.load(config.getAssociatedTaxonNameUuid());
1494         }
1495         occurrences.addAll(dao.findOccurrencesUuidAndTitleCache(config.getClazz(),
1496                 config.getTitleSearchString(), config.getSignificantIdentifier(),
1497                 config.getSpecimenType(), taxon, taxonName, config.getMatchMode(), null, null,
1498                 config.getOrderHints()));
1499         long count = Integer.valueOf(occurrences.size()).longValue();
1500         return new DefaultPagerImpl<>(config.getPageNumber(), count, config.getPageSize(), occurrences);
1501     }
1502
1503     @Override
1504     public List<DerivedUnitDTO> findByTitleDerivedUnitDTO(FindOccurrencesConfigurator config) {
1505         Taxon taxon = null;
1506         if(config.getAssociatedTaxonUuid()!=null){
1507             TaxonBase<?> taxonBase = taxonService.load(config.getAssociatedTaxonUuid());
1508             if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1509                 taxon = CdmBase.deproxy(taxonBase, Taxon.class);
1510             }
1511         }
1512         TaxonName taxonName = null;
1513         if(config.getAssociatedTaxonNameUuid()!=null){
1514             taxonName = nameService.load(config.getAssociatedTaxonNameUuid());
1515         }
1516         List<DerivedUnit> occurrences = new ArrayList<>();
1517         occurrences.addAll(dao.findOccurrences(DerivedUnit.class,
1518                 config.getTitleSearchString(), config.getSignificantIdentifier(),
1519                 config.getSpecimenType(), taxon, taxonName, config.getMatchMode(), null, null,
1520                 config.getOrderHints(), null));
1521
1522         List<DerivedUnitDTO> dtos = new ArrayList<>();
1523         occurrences.forEach(derivedUnit->dtos.add(assembleDerivedUnitDTO(derivedUnit)));
1524         return dtos;
1525     }
1526
1527     @Override
1528     public <S extends SpecimenOrObservationBase> Pager<S> findByTitle(
1529             IIdentifiableEntityServiceConfigurator<S> config) {
1530         if (config instanceof FindOccurrencesConfigurator) {
1531             FindOccurrencesConfigurator occurrenceConfig = (FindOccurrencesConfigurator) config;
1532             List<SpecimenOrObservationBase> occurrences = new ArrayList<>();
1533             Taxon taxon = null;
1534             if(occurrenceConfig.getAssociatedTaxonUuid()!=null){
1535                 TaxonBase<?> taxonBase = taxonService.load(occurrenceConfig.getAssociatedTaxonUuid());
1536                 if(taxonBase.isInstanceOf(Taxon.class)){
1537                     taxon = HibernateProxyHelper.deproxy(taxonBase, Taxon.class);
1538                 }
1539             }
1540             TaxonName taxonName = null;
1541             if(occurrenceConfig.getAssociatedTaxonNameUuid()!=null){
1542                 taxonName = nameService.load(occurrenceConfig.getAssociatedTaxonNameUuid());
1543             }
1544             List<? extends SpecimenOrObservationBase> foundOccurrences = dao.findOccurrences(occurrenceConfig.getClazz(),
1545                     occurrenceConfig.getTitleSearchString(), occurrenceConfig.getSignificantIdentifier(),
1546                     occurrenceConfig.getSpecimenType(), taxon, taxonName, occurrenceConfig.getMatchMode(), null, null,
1547                     occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1548             occurrences.addAll(foundOccurrences);
1549             occurrences = filterOccurencesByAssignmentAndHierarchy(occurrenceConfig, occurrences, taxon, taxonName);
1550
1551             long count = Integer.valueOf(occurrences.size()).longValue();
1552             return new DefaultPagerImpl<>(config.getPageNumber(), count, config.getPageSize(), (List<S>)occurrences);
1553         }
1554         return super.findByTitle(config);
1555     }
1556
1557     private List<SpecimenOrObservationBase> filterOccurencesByAssignmentAndHierarchy(
1558             FindOccurrencesConfigurator occurrenceConfig, List<SpecimenOrObservationBase> occurrences, Taxon taxon,
1559             TaxonName taxonName) {
1560         //filter out (un-)assigned specimens
1561         if(taxon==null && taxonName==null){
1562             AssignmentStatus assignmentStatus = occurrenceConfig.getAssignmentStatus();
1563             List<SpecimenOrObservationBase> specimenWithAssociations = new ArrayList<>();
1564             if(!assignmentStatus.equals(AssignmentStatus.ALL_SPECIMENS)){
1565                 for (SpecimenOrObservationBase<?> specimenOrObservationBase : occurrences) {
1566                     boolean includeUnpublished = true;  //TODO not sure if this is correct, maybe we have to propagate publish flag to higher methods.
1567                     Collection<TaxonBase<?>> associatedTaxa = listAssociatedTaxa(specimenOrObservationBase,
1568                             includeUnpublished, null, null, null, null);
1569                     if(!associatedTaxa.isEmpty()){
1570                         specimenWithAssociations.add(specimenOrObservationBase);
1571                     }
1572                 }
1573             }
1574             if(assignmentStatus.equals(AssignmentStatus.UNASSIGNED_SPECIMENS)){
1575                 occurrences.removeAll(specimenWithAssociations);
1576             }
1577             if(assignmentStatus.equals(AssignmentStatus.ASSIGNED_SPECIMENS)){
1578                 occurrences = new ArrayList<>(specimenWithAssociations);
1579             }
1580         }
1581         // indirectly associated specimens
1582         if(occurrenceConfig.isRetrieveIndirectlyAssociatedSpecimens()){
1583             List<SpecimenOrObservationBase> indirectlyAssociatedOccurrences = new ArrayList<>(occurrences);
1584             for (SpecimenOrObservationBase<?> specimen : occurrences) {
1585                 List<SpecimenOrObservationBase<?>> allHierarchyDerivates = getAllHierarchyDerivatives(specimen);
1586                 for (SpecimenOrObservationBase<?> specimenOrObservationBase : allHierarchyDerivates) {
1587                     if(!occurrences.contains(specimenOrObservationBase)){
1588                         indirectlyAssociatedOccurrences.add(specimenOrObservationBase);
1589                     }
1590                 }
1591             }
1592             occurrences = indirectlyAssociatedOccurrences;
1593         }
1594         return occurrences;
1595     }
1596
1597     @Override
1598     public List<SpecimenOrObservationBase<?>> getAllHierarchyDerivatives(SpecimenOrObservationBase<?> specimen){
1599         List<SpecimenOrObservationBase<?>> allHierarchyDerivatives = new ArrayList<>();
1600         Collection<FieldUnit> fieldUnits = findFieldUnits(specimen.getUuid(), null);
1601         if(fieldUnits.isEmpty()){
1602             allHierarchyDerivatives.add(specimen);
1603             allHierarchyDerivatives.addAll(getAllChildDerivatives(specimen));
1604         }
1605         else{
1606             for (FieldUnit fieldUnit : fieldUnits) {
1607                 allHierarchyDerivatives.add(fieldUnit);
1608                 allHierarchyDerivatives.addAll(getAllChildDerivatives(fieldUnit));
1609             }
1610         }
1611         return allHierarchyDerivatives;
1612     }
1613
1614     @Override
1615     public List<DerivedUnit> getAllChildDerivatives(UUID specimenUuid){
1616         return getAllChildDerivatives(load(specimenUuid));
1617     }
1618
1619     @Override
1620     public List<DerivedUnit> getAllChildDerivatives(SpecimenOrObservationBase<?> specimen){
1621         if (specimen == null){
1622             return null;
1623         }
1624         List<DerivedUnit> childDerivate = new ArrayList<>();
1625         Set<DerivationEvent> derivationEvents = specimen.getDerivationEvents();
1626         for (DerivationEvent derivationEvent : derivationEvents) {
1627             Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1628             for (DerivedUnit derivedUnit : derivatives) {
1629                 childDerivate.add(derivedUnit);
1630                 childDerivate.addAll(getAllChildDerivatives(derivedUnit.getUuid()));
1631             }
1632         }
1633         return childDerivate;
1634     }
1635
1636     @Override
1637     public long countOccurrences(IIdentifiableEntityServiceConfigurator<SpecimenOrObservationBase> config){
1638         return countByTitle(config);
1639     }
1640
1641     @Override
1642     public List<FieldUnit> findFieldUnitsForGatheringEvent(UUID gatheringEventUuid) {
1643         return dao.findFieldUnitsForGatheringEvent(gatheringEventUuid, null, null, null, null);
1644     }
1645
1646     @Override
1647     public List<Point> findPointsForFieldUnitList(List<UUID> fieldUnitUuids) {
1648         return dao.findPointsForFieldUnitList(fieldUnitUuids);
1649     }
1650 }