cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/OccurrenceServiceImpl.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9 package eu.etaxonomy.cdm.api.service;
  10
  11 import java.io.IOException;
  12 import java.util.ArrayList;
  13 import java.util.Arrays;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.Comparator;
  17 import java.util.EnumSet;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24 import java.util.UUID;
  25 import java.util.stream.Collectors;
  26
  27 import org.apache.commons.lang3.StringUtils;
  28 import org.apache.logging.log4j.LogManager;
  29 import org.apache.logging.log4j.Logger;
  30 import org.apache.lucene.queryparser.classic.ParseException;
  31 import org.apache.lucene.search.BooleanClause.Occur;
  32 import org.apache.lucene.search.BooleanQuery.Builder;
  33 import org.apache.lucene.search.SortField;
  34 import org.apache.lucene.search.grouping.TopGroups;
  35 import org.apache.lucene.util.BytesRef;
  36 import org.hibernate.TransientObjectException;
  37 import org.springframework.beans.factory.annotation.Autowired;
  38 import org.springframework.dao.DataRetrievalFailureException;
  39 import org.springframework.stereotype.Service;
  40 import org.springframework.transaction.annotation.Transactional;
  41
  42 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  43 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeConfigurator;
  44 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeNotSupportedException;
  45 import eu.etaxonomy.cdm.api.service.UpdateResult.Status;
  46 import eu.etaxonomy.cdm.api.service.config.DeleteConfiguratorBase;
  47 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  48 import eu.etaxonomy.cdm.api.service.config.IIdentifiableEntityServiceConfigurator;
  49 import eu.etaxonomy.cdm.api.service.config.SpecimenDeleteConfigurator;
  50 import eu.etaxonomy.cdm.api.service.dto.DNASampleDTO;
  51 import eu.etaxonomy.cdm.api.service.dto.DerivedUnitDTO;
  52 import eu.etaxonomy.cdm.api.service.dto.FieldUnitDTO;
  53 import eu.etaxonomy.cdm.api.service.dto.MediaDTO;
  54 import eu.etaxonomy.cdm.api.service.dto.RectangleDTO;
  55 import eu.etaxonomy.cdm.api.service.dto.SpecimenOrObservationBaseDTO;
  56 import eu.etaxonomy.cdm.api.service.dto.SpecimenOrObservationDTOFactory;
  57 import eu.etaxonomy.cdm.api.service.exception.ReferencedObjectUndeletableException;
  58 import eu.etaxonomy.cdm.api.service.molecular.ISequenceService;
  59 import eu.etaxonomy.cdm.api.service.pager.Pager;
  60 import eu.etaxonomy.cdm.api.service.pager.impl.AbstractPagerImpl;
  61 import eu.etaxonomy.cdm.api.service.pager.impl.DefaultPagerImpl;
  62 import eu.etaxonomy.cdm.api.service.search.ILuceneIndexToolProvider;
  63 import eu.etaxonomy.cdm.api.service.search.ISearchResultBuilder;
  64 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  65 import eu.etaxonomy.cdm.api.service.search.LuceneSearch;
  66 import eu.etaxonomy.cdm.api.service.search.QueryFactory;
  67 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  68 import eu.etaxonomy.cdm.api.service.search.SearchResultBuilder;
  69 import eu.etaxonomy.cdm.api.util.TaxonRelationshipEdge;
  70 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  71 import eu.etaxonomy.cdm.compare.common.PartialComparator;
  72 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  73 import eu.etaxonomy.cdm.model.CdmBaseType;
  74 import eu.etaxonomy.cdm.model.common.CdmBase;
  75 import eu.etaxonomy.cdm.model.common.Language;
  76 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  77 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  78 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  79 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  80 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  81 import eu.etaxonomy.cdm.model.location.Point;
  82 import eu.etaxonomy.cdm.model.media.Media;
  83 import eu.etaxonomy.cdm.model.molecular.AmplificationResult;
  84 import eu.etaxonomy.cdm.model.molecular.DnaSample;
  85 import eu.etaxonomy.cdm.model.molecular.Sequence;
  86 import eu.etaxonomy.cdm.model.molecular.SingleRead;
  87 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  88 import eu.etaxonomy.cdm.model.name.TaxonName;
  89 import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
  90 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  91 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  92 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  93 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  94 import eu.etaxonomy.cdm.model.occurrence.MediaSpecimen;
  95 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  96 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  97 import eu.etaxonomy.cdm.model.taxon.Taxon;
  98 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  99 import eu.etaxonomy.cdm.persistence.dao.initializer.AbstractBeanInitializer;
 100 import eu.etaxonomy.cdm.persistence.dao.occurrence.IOccurrenceDao;
 101 import eu.etaxonomy.cdm.persistence.dto.SpecimenNodeWrapper;
 102 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
 103 import eu.etaxonomy.cdm.persistence.query.AssignmentStatus;
 104 import eu.etaxonomy.cdm.persistence.query.OrderHint;
 105 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
 106
 107 /**
 108  * @author a.babadshanjan
 109  * @since 01.09.2008
 110  */
 111 @Service
 112 @Transactional(readOnly = true)
 113 public class OccurrenceServiceImpl
 114         extends IdentifiableServiceBase<SpecimenOrObservationBase, IOccurrenceDao>
 115         implements IOccurrenceService {
 116
 117     private static final Logger logger = LogManager.getLogger();
 118
 119     @Autowired
 120     private IDescriptionService descriptionService;
 121
 122     @Autowired
 123     private INameService nameService;
 124
 125     @Autowired
 126     private IEventBaseService eventService;
 127
 128     @Autowired
 129     private ITaxonService taxonService;
 130
 131     @Autowired
 132     private ISequenceService sequenceService;
 133
 134     @Autowired
 135     private AbstractBeanInitializer<?> beanInitializer;
 136
 137     @Autowired
 138     private ILuceneIndexToolProvider luceneIndexToolProvider;
 139
 140     public OccurrenceServiceImpl() {
 141         logger.debug("Load OccurrenceService Bean");
 142     }
 143
 144     @Override
 145     @Transactional(readOnly = false)
 146     public UpdateResult updateCaches(Class<? extends SpecimenOrObservationBase> clazz, Integer stepSize, IIdentifiableEntityCacheStrategy<SpecimenOrObservationBase> cacheStrategy, IProgressMonitor monitor) {
 147         if (clazz == null) {
 148             clazz = SpecimenOrObservationBase.class;
 149         }
 150         return super.updateCachesImpl(clazz, stepSize, cacheStrategy, monitor);
 151     }
 152
 153     @Override
 154     @Autowired
 155     protected void setDao(IOccurrenceDao dao) {
 156         this.dao = dao;
 157     }
 158
 159     @Override
 160     public Pager<DerivationEvent> getDerivationEvents(SpecimenOrObservationBase occurence, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 161         long numberOfResults = dao.countDerivationEvents(occurence);
 162
 163         List<DerivationEvent> results = new ArrayList<>();
 164         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 165             results = dao.getDerivationEvents(occurence, pageSize, pageNumber, propertyPaths);
 166         }
 167
 168         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 169     }
 170
 171     @Override
 172     public long countDeterminations(SpecimenOrObservationBase occurence, TaxonBase taxonbase) {
 173         return dao.countDeterminations(occurence, taxonbase);
 174     }
 175
 176     @Override
 177     public Pager<DeterminationEvent> getDeterminations(SpecimenOrObservationBase occurrence, TaxonBase taxonBase,
 178             Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 179         long numberOfResults = dao.countDeterminations(occurrence, taxonBase);
 180
 181         List<DeterminationEvent> results = new ArrayList<>();
 182         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 183             results = dao.getDeterminations(occurrence, taxonBase, pageSize, pageNumber, propertyPaths);
 184         }
 185
 186         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 187     }
 188
 189     @Override
 190     public Pager<Media> getMedia(SpecimenOrObservationBase occurence, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 191         long numberOfResults = dao.countMedia(occurence);
 192
 193         List<Media> results = new ArrayList<>();
 194         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 195             results = dao.getMedia(occurence, pageSize, pageNumber, propertyPaths);
 196         }
 197
 198         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 199     }
 200
 201     @Override
 202     public Pager<MediaDTO> getMediaDTOs(SpecimenOrObservationBase<?> occurence, Integer pageSize, Integer pageNumber) {
 203         long numberOfResults = dao.countMedia(occurence);
 204
 205         List<Media> results = new ArrayList<>();
 206         if(AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)) {
 207             results = dao.getMedia(occurence, pageSize, pageNumber, null);
 208         }
 209         List<MediaDTO> mediaDTOs = results.stream()
 210                 .map(m -> MediaDTO.fromEntity(m))
 211                 .flatMap(dtos -> dtos.stream())
 212                 .collect(Collectors.toList()
 213                 );
 214         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, mediaDTOs);
 215     }
 216     @Override
 217     public Pager<Media> getMediaInHierarchy(SpecimenOrObservationBase<?> rootOccurence, boolean collectOriginalMedia, boolean collectDerivateMedia, Integer pageSize,
 218             Integer pageNumber, List<String> propertyPaths) {
 219          List<Media> media = new ArrayList<>();
 220          //media specimens
 221          if(rootOccurence.isInstanceOf(MediaSpecimen.class)){
 222              MediaSpecimen mediaSpecimen = HibernateProxyHelper.deproxy(rootOccurence, MediaSpecimen.class);
 223              media.add(mediaSpecimen.getMediaSpecimen());
 224          }
 225          // pherograms & gelPhotos
 226          if (rootOccurence.isInstanceOf(DnaSample.class)) {
 227              DnaSample dnaSample = CdmBase.deproxy(rootOccurence, DnaSample.class);
 228              Set<Sequence> sequences = dnaSample.getSequences();
 229              //we do show only those gelPhotos which lead to a consensus sequence
 230              for (Sequence sequence : sequences) {
 231                  Set<Media> dnaRelatedMedia = new HashSet<>();
 232                  for (SingleRead singleRead : sequence.getSingleReads()){
 233                      AmplificationResult amplification = singleRead.getAmplificationResult();
 234                      dnaRelatedMedia.add(amplification.getGelPhoto());
 235                      dnaRelatedMedia.add(singleRead.getPherogram());
 236                      dnaRelatedMedia.remove(null);
 237                  }
 238                  media.addAll(dnaRelatedMedia);
 239              }
 240          }
 241          if(rootOccurence.isInstanceOf(DerivedUnit.class)){
 242              DerivedUnit derivedUnit = HibernateProxyHelper.deproxy(rootOccurence, DerivedUnit.class);
 243              if (collectDerivateMedia) {
 244                      for (DerivationEvent derivationEvent : derivedUnit.getDerivationEvents()) {
 245                          for (DerivedUnit childDerivative : derivationEvent.getDerivatives()) {
 246                                  //collectOriginalMedia should only called for the first derived unit
 247                              media.addAll(getMediaInHierarchy(childDerivative, false, true, pageSize, pageNumber, propertyPaths).getRecords());
 248                          }
 249                      }
 250              }
 251              if (collectOriginalMedia) {
 252                  for (SpecimenOrObservationBase original : derivedUnit.getOriginals()) {
 253                                  //collect media to the top of the tree
 254                              media.addAll(getMediaInHierarchy(original, true, false, pageSize, pageNumber, propertyPaths).getRecords());
 255                      }
 256                  }
 257          }
 258
 259
 260          return new DefaultPagerImpl<>(pageNumber, Long.valueOf(media.size()), pageSize, media);
 261     }
 262
 263
 264     @Override
 265     public Pager<Media> getMediaInHierarchy(SpecimenOrObservationBase<?> rootOccurence, Integer pageSize,
 266             Integer pageNumber, List<String> propertyPaths) {
 267         return getMediaInHierarchy(rootOccurence, false, true, pageSize,
 268                 pageNumber, propertyPaths);
 269
 270     }
 271
 272     @Override
 273     public Pager<SpecimenOrObservationBase> list(Class<? extends SpecimenOrObservationBase> type, TaxonName determinedAs,
 274             Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 275         long numberOfResults = dao.count(type, determinedAs);
 276         @SuppressWarnings("rawtypes")
 277         List<SpecimenOrObservationBase> results = new ArrayList<>();
 278         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 279             results = dao.list(type, determinedAs, pageSize, pageNumber, orderHints, propertyPaths);
 280         }
 281         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 282     }
 283
 284     @Override
 285     public Pager<SpecimenOrObservationBase> list(Class<? extends SpecimenOrObservationBase> type, TaxonBase determinedAs,
 286             Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 287         long numberOfResults = dao.count(type, determinedAs);
 288         @SuppressWarnings("rawtypes")
 289         List<SpecimenOrObservationBase> results = new ArrayList<>();
 290         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 291             results = dao.list(type, determinedAs, pageSize, pageNumber, orderHints, propertyPaths);
 292         }
 293         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 294     }
 295
 296     @Override
 297     public List<UuidAndTitleCache<DerivedUnit>> getDerivedUnitUuidAndTitleCache(Integer limit, String pattern) {
 298         return dao.getDerivedUnitUuidAndTitleCache(limit, pattern);
 299     }
 300
 301     @Override
 302     public List<UuidAndTitleCache<FieldUnit>> getFieldUnitUuidAndTitleCache() {
 303         return dao.getFieldUnitUuidAndTitleCache();
 304     }
 305
 306     @Override
 307     public DerivedUnitFacade getDerivedUnitFacade(DerivedUnit derivedUnit, List<String> derivedUnitFacadeInitStrategy) throws DerivedUnitFacadeNotSupportedException {
 308         derivedUnit = (DerivedUnit) dao.load(derivedUnit.getUuid(), null);
 309         DerivedUnitFacadeConfigurator config = DerivedUnitFacadeConfigurator.NewInstance();
 310         config.setThrowExceptionForNonSpecimenPreservationMethodRequest(false);
 311         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(derivedUnit, config);
 312         beanInitializer.initialize(derivedUnitFacade, derivedUnitFacadeInitStrategy);
 313         return derivedUnitFacade;
 314     }
 315
 316     @Override
 317     public List<DerivedUnitFacade> listDerivedUnitFacades(
 318             DescriptionBase description, List<String> derivedUnitFacadeInitStrategy) {
 319
 320         List<DerivedUnitFacade> derivedUnitFacadeList = new ArrayList<>();
 321         IndividualsAssociation tempIndividualsAssociation;
 322         SpecimenOrObservationBase tempSpecimenOrObservationBase;
 323         List<IndividualsAssociation> elements = descriptionService.listDescriptionElements(description, null, IndividualsAssociation.class, null, 0, Arrays.asList(new String []{"associatedSpecimenOrObservation"}));
 324         for (IndividualsAssociation element : elements) {
 325             tempIndividualsAssociation = HibernateProxyHelper.deproxy(element, IndividualsAssociation.class);
 326             if (tempIndividualsAssociation.getAssociatedSpecimenOrObservation() != null) {
 327                 tempSpecimenOrObservationBase = HibernateProxyHelper.deproxy(tempIndividualsAssociation.getAssociatedSpecimenOrObservation(), SpecimenOrObservationBase.class);
 328                 if (tempSpecimenOrObservationBase.isInstanceOf(DerivedUnit.class)) {
 329                     try {
 330                         derivedUnitFacadeList.add(DerivedUnitFacade.NewInstance(HibernateProxyHelper.deproxy(tempSpecimenOrObservationBase, DerivedUnit.class)));
 331                     } catch (DerivedUnitFacadeNotSupportedException e) {
 332                         logger.warn(tempIndividualsAssociation.getAssociatedSpecimenOrObservation().getTitleCache() + " : " + e.getMessage());
 333                     }
 334                 }
 335             }
 336         }
 337
 338         beanInitializer.initializeAll(derivedUnitFacadeList, derivedUnitFacadeInitStrategy);
 339
 340         return derivedUnitFacadeList;
 341     }
 342
 343
 344     @Override
 345     public <T extends SpecimenOrObservationBase> List<T> listByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 346             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 347
 348         return pageByAssociatedTaxon(type, includeRelationships, associatedTaxon, maxDepth, pageSize, pageNumber, orderHints, propertyPaths).getRecords();
 349     }
 350
 351     @Override
 352     public Collection<SpecimenNodeWrapper> listUuidAndTitleCacheByAssociatedTaxon(List<UUID> taxonNodeUuids,
 353             Integer limit, Integer start) {
 354         return dao.listUuidAndTitleCacheByAssociatedTaxon(taxonNodeUuids, limit, start);
 355         }
 356
 357     @Override
 358     @Deprecated
 359     public Collection<FieldUnit> listFieldUnitsByAssociatedTaxon(Taxon associatedTaxon, List<OrderHint> orderHints, List<String> propertyPaths) {
 360         return pageRootUnitsByAssociatedTaxon(FieldUnit.class, null, associatedTaxon, null, null, null, null, propertyPaths).getRecords();
 361     }
 362
 363     @Override
 364     public <T extends SpecimenOrObservationBase> Collection<T> listRootUnitsByAssociatedTaxon(Class<T> type, Taxon associatedTaxon, List<OrderHint> orderHints, List<String> propertyPaths) {
 365         return pageRootUnitsByAssociatedTaxon(type, null, associatedTaxon, null, null, null, null, propertyPaths).getRecords();
 366     }
 367
 368     @Override
 369     public <T extends SpecimenOrObservationBase> Pager<T> pageRootUnitsByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 370             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints,
 371             List<String> propertyPaths) {
 372
 373         if (!getSession().contains(associatedTaxon)) {
 374             associatedTaxon = (Taxon) taxonService.load(associatedTaxon.getUuid());
 375         }
 376
 377         // gather the IDs of all relevant root units
 378         Set<UUID> rootUnitUuids = new HashSet<>();
 379         List<SpecimenOrObservationBase> records = listByAssociatedTaxon(null, includeRelationships, associatedTaxon, maxDepth, null, null, orderHints, propertyPaths);
 380         for (SpecimenOrObservationBase<?> specimen : records) {
 381             for (SpecimenOrObservationBase<?> rootUnit : findRootUnits(specimen.getUuid(), null)) {
 382                 if(type == null || type.isAssignableFrom(rootUnit.getClass())) {
 383                     rootUnitUuids.add(rootUnit.getUuid());
 384                 }
 385             }
 386         }
 387         long totalCount = rootUnitUuids.size();
 388         //dao.list() does the paging of the field units. Passing the field units directly to the Pager would not work
 389         List<SpecimenOrObservationBase> rootUnits = dao.list(rootUnitUuids, pageSize, pageNumber, orderHints, propertyPaths);
 390         List<T> castedUnits = new ArrayList<>(rootUnits.size());
 391         for(SpecimenOrObservationBase<?> sob : rootUnits) {
 392             // this cast should be save since the uuids have been filtered by type above
 393             castedUnits.add((T)sob);
 394         }
 395         return new DefaultPagerImpl<>(pageNumber, totalCount, pageSize, castedUnits);
 396     }
 397
 398     @Override
 399     public FieldUnitDTO assembleFieldUnitDTO(FieldUnit fieldUnit) {
 400
 401         if (!getSession().contains(fieldUnit)) {
 402             fieldUnit = (FieldUnit) load(fieldUnit.getUuid());
 403         }
 404         // FIXME the filter for SpecimenOrObservationType.PreservedSpecimen has been preserved from the former implementation (see commit 07e3f63c7d  and older)
 405         // it is questionable if this filter makes sense for all use cases or if it is only a sensible default for the
 406         // compressed specimen table in the cdm-dataportal (see #6816, #6870)
 407         EnumSet<SpecimenOrObservationType> typeIncludeFilter = EnumSet.of(SpecimenOrObservationType.PreservedSpecimen);
 408         FieldUnitDTO fieldUnitDTO = FieldUnitDTO.fromEntity(fieldUnit, null, typeIncludeFilter);
 409         return fieldUnitDTO;
 410     }
 411
 412
 413     @Override
 414     @Transactional
 415     public DerivedUnitDTO assembleDerivedUnitDTO(DerivedUnit derivedUnit) {
 416
 417         if (!getSession().contains(derivedUnit)) {
 418             derivedUnit = (DerivedUnit) load(derivedUnit.getUuid());
 419         }
 420         DerivedUnitDTO derivedUnitDTO = DerivedUnitDTO.fromEntity(derivedUnit, null, null);
 421
 422         // individuals associations
 423         Collection<IndividualsAssociation> individualsAssociations = listIndividualsAssociations(derivedUnit, null, null, null, null);
 424         if(individualsAssociations != null) {
 425             for (IndividualsAssociation individualsAssociation : individualsAssociations) {
 426                 if (individualsAssociation.getInDescription() != null) {
 427                     if (individualsAssociation.getInDescription().isInstanceOf(TaxonDescription.class)) {
 428                         TaxonDescription taxonDescription = HibernateProxyHelper.deproxy(individualsAssociation.getInDescription(), TaxonDescription.class);
 429                         Taxon taxon = taxonDescription.getTaxon();
 430                         if (taxon != null) {
 431                             derivedUnitDTO.addAssociatedTaxon(taxon);
 432                         }
 433                     }
 434                 }
 435             }
 436         }
 437
 438         return derivedUnitDTO;
 439     }
 440
 441     @Override
 442
 443     public String getMostSignificantIdentifier(UUID derivedUnitUuid) {
 444         return dao.findMostSignificantIdentifier(derivedUnitUuid);
 445     }
 446
 447     /**
 448      * TODO there is a very similar function in {@link SpecimenOrObservationBaseDTO#assembleDerivative}.
 449      * If possible we should avoid using this function here by the method in <code>SpecimenOrObservationBaseDTO</code>.
 450      */
 451     private Set<DerivedUnitDTO> getDerivedUnitDTOsFor(SpecimenOrObservationBaseDTO specimenDto, DerivedUnit specimen,
 452             HashMap<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 453
 454         Set<DerivedUnitDTO> derivedUnits = new HashSet<>();
 455         for (DerivationEvent derivationEvent : specimen.getDerivationEvents()) {
 456             for (DerivedUnit derivative : derivationEvent.getDerivatives()) {
 457                 if (!alreadyCollectedSpecimen.containsKey(specimenDto.getUuid())){
 458                     DerivedUnitDTO dto = (DerivedUnitDTO) SpecimenOrObservationDTOFactory.fromEntity(derivative, 0);
 459                     alreadyCollectedSpecimen.put(dto.getUuid(), dto);
 460                     dto.addAllDerivatives(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 461                     derivedUnits.add(dto);
 462                 } else {
 463                     if(alreadyCollectedSpecimen.get(specimenDto.getUuid()).getDerivatives().isEmpty() && !derivative.getDerivationEvents().isEmpty()) {
 464                         // we need to add the missing derivatives!
 465                         SpecimenOrObservationBaseDTO dto = alreadyCollectedSpecimen.get(specimenDto.getUuid());
 466                         alreadyCollectedSpecimen.get(specimenDto.getUuid()).addAllDerivatives(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 467                     }
 468                 }
 469             }
 470         }
 471         return derivedUnits;
 472     }
 473
 474 //    private Set<DerivateDTO> getDerivedUnitDTOsFor(DerivateDTO specimenDto, DerivedUnit specimen, HashMap<UUID, DerivateDTO> alreadyCollectedSpecimen) {
 475 //        Set<DerivateDTO> derivedUnits = new HashSet<>();
 476 ////        load
 477 //        for (DerivationEvent derivationEvent : specimen.getDerivationEvents()) {
 478 //            for (DerivedUnit derivative : derivationEvent.getDerivatives()) {
 479 //                if (!alreadyCollectedSpecimen.containsKey(specimenDto.getUuid())){
 480 //                    PreservedSpecimenDTO dto;
 481 //                    if (derivative instanceof DnaSample){
 482 //                        dto = DNASampleDTO.newInstance(derivative);
 483 //                    }else{
 484 //                        dto = PreservedSpecimenDTO.newInstance(derivative);
 485 //                    }
 486 //                    alreadyCollectedSpecimen.put(dto.getUuid(), dto);
 487 //                    dto.addAllDerivates(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 488 //                    derivedUnits.add(dto);
 489 //                }
 490 //            }
 491 //        }
 492 //        return derivedUnits;
 493 //    }
 494
 495
 496     @SuppressWarnings("unchecked")
 497     @Override
 498     public <T extends SpecimenOrObservationBase> Pager<T> pageByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includedRelationships,
 499             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 500
 501         Set<Taxon> taxa = new HashSet<>();
 502         Set<Integer> occurrenceIds = new HashSet<>();
 503         List<T> occurrences = new ArrayList<>();
 504         boolean includeUnpublished = INCLUDE_UNPUBLISHED;
 505
 506         // Integer limit = PagerUtils.limitFor(pageSize);
 507         // Integer start = PagerUtils.startFor(pageSize, pageNumber);
 508
 509         if (!getSession().contains(associatedTaxon)) {
 510             associatedTaxon = (Taxon) taxonService.load(associatedTaxon.getUuid());
 511         }
 512
 513         if (includedRelationships != null) {
 514             taxa = taxonService.listRelatedTaxa(associatedTaxon, includedRelationships, maxDepth, includeUnpublished, null, null, propertyPaths);
 515         }
 516
 517         taxa.add(associatedTaxon);
 518
 519         for (Taxon taxon : taxa) {
 520             List<T> perTaxonOccurrences = dao.listByAssociatedTaxon(type, taxon, null, null, orderHints, propertyPaths);
 521             for (SpecimenOrObservationBase<?> o : perTaxonOccurrences) {
 522                 occurrenceIds.add(o.getId());
 523             }
 524         }
 525         occurrences = (List<T>) dao.loadList(occurrenceIds, null, propertyPaths);
 526
 527         return new DefaultPagerImpl<>(pageNumber, Long.valueOf(occurrences.size()), pageSize, occurrences);
 528
 529     }
 530
 531     @Override
 532     public <T extends SpecimenOrObservationBase> Pager<T> pageByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 533             String taxonUUID, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 534
 535         UUID uuid = UUID.fromString(taxonUUID);
 536         Taxon taxon = (Taxon) taxonService.load(uuid);
 537         return pageByAssociatedTaxon(type, includeRelationships, taxon, maxDepth, pageSize, pageNumber, orderHints, propertyPaths);
 538
 539     }
 540
 541     @Override
 542     @Transactional
 543     public List<SpecimenOrObservationBaseDTO> listRootUnitDTOsByAssociatedTaxon(Set<TaxonRelationshipEdge> includedRelationships,
 544             UUID associatedTaxonUuid, List<String> propertyPaths) {
 545
 546         Set<Taxon> taxa = new HashSet<>();
 547         Set<SpecimenOrObservationBaseDTO> rootUnitDTOs = new HashSet<>();
 548         boolean includeUnpublished = INCLUDE_UNPUBLISHED;
 549
 550         Taxon associatedTaxon = (Taxon) taxonService.load(associatedTaxonUuid);
 551         if (includedRelationships != null) {
 552             taxa = taxonService.listRelatedTaxa(associatedTaxon, includedRelationships, null, includeUnpublished, null, null, null);
 553         }
 554         taxa.add(associatedTaxon);
 555
 556         HashMap<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedUnits = new HashMap<>();
 557         for (Taxon taxon : taxa) {
 558             // TODO there might be a good potential to speed up the whole processing by collecting all entities first
 559             // and to create the DTOs in a second step
 560             Set<SpecimenOrObservationBase> perTaxonOccurrences = dao.listByAssociatedTaxon(null, taxon, null, null, null, propertyPaths)
 561                     .stream()
 562                     .map(u -> HibernateProxyHelper.deproxy(u, SpecimenOrObservationBase.class))
 563                     .collect(Collectors.toSet());
 564             for (SpecimenOrObservationBase<?> unit : perTaxonOccurrences) {
 565                 unit = HibernateProxyHelper.deproxy(unit);
 566                 if (unit instanceof DerivedUnit){
 567                     DerivedUnitDTO derivativeDTO;
 568                     if (!alreadyCollectedUnits.containsKey(unit.getUuid())){
 569                         DerivedUnit derivedUnit = (DerivedUnit)unit;
 570                         boolean isAssociated = true;
 571                         for (DeterminationEvent determination:derivedUnit.getDeterminations()) {
 572                                 if (determination.getTaxonName() != null && determination.getTaxonName().equals(taxon.getName()) || taxon.equals(determination.getTaxon())){
 573                                         isAssociated = true;
 574                                         break;
 575                                 }else {
 576                                         isAssociated = false;
 577                                 }
 578                         }
 579
 580                         if (!isAssociated) {
 581                                 continue;
 582                         }
 583                         derivativeDTO = (DerivedUnitDTO) SpecimenOrObservationDTOFactory.fromEntity(derivedUnit, null);
 584                         alreadyCollectedUnits.put(derivativeDTO.getUuid(), derivativeDTO);
 585                         //derivativeDTO.addAllDerivatives(getDerivedUnitDTOsFor(derivativeDTO, derivedUnit, alreadyCollectedUnits));
 586                     }
 587                     derivativeDTO = (DerivedUnitDTO) alreadyCollectedUnits.get(unit.getUuid());
 588                     rootUnitDTOs.addAll(findRootUnitDTOs(derivativeDTO, alreadyCollectedUnits));
 589                 } else {
 590                     // only other option is FieldUnit
 591                     rootUnitDTOs.add(FieldUnitDTO.fromEntity((FieldUnit)unit, 0, null));
 592                 }
 593             }
 594         }
 595
 596         List<SpecimenOrObservationBaseDTO> orderdDTOs = new ArrayList<>(rootUnitDTOs);
 597         // TODO order dtos by date can only be done by string comparison
 598         // the FieldUnitDTO.date needs to be a Partial object for sensible ordering
 599         Collections.sort(orderdDTOs, new Comparator<SpecimenOrObservationBaseDTO>() {
 600
 601             @Override
 602             public int compare(SpecimenOrObservationBaseDTO o1, SpecimenOrObservationBaseDTO o2) {
 603                 if(o1 instanceof FieldUnitDTO && o2 instanceof FieldUnitDTO) {
 604                     FieldUnitDTO fu1 = (FieldUnitDTO)o1;
 605                     FieldUnitDTO fu2 = (FieldUnitDTO)o2;
 606                     //TODO if we want null values and values with missing year as smallest we should use
 607                     //PartialComparator.INSTANCE_NULL_SMALLEST() here
 608                     return PartialComparator.INSTANCE().compare(fu1.getDate(), fu2.getDate());
 609                 }
 610                 if(o1 instanceof DerivedUnitDTO && o2 instanceof DerivedUnitDTO) {
 611                     SpecimenOrObservationBaseDTO du1 = o1;
 612                     SpecimenOrObservationBaseDTO du2 = o2;
 613                     return StringUtils.compare(du1.getLabel(), du2.getLabel());
 614                  }
 615                 if(o1 instanceof FieldUnitDTO && o2 instanceof DerivedUnitDTO) {
 616                     return -1;
 617                 } else {
 618                     return 1;
 619                 }
 620             }
 621
 622         });
 623
 624         return orderdDTOs;
 625     }
 626
 627     @Override
 628     @Transactional
 629     @Deprecated
 630     public  SpecimenOrObservationBaseDTO findByAccessionNumber(String accessionNumberString, List<OrderHint> orderHints)  {
 631         return findByAccessionNumber(accessionNumberString, orderHints);
 632     }
 633
 634     @Override
 635     @Transactional
 636     public  SpecimenOrObservationBaseDTO findByGeneticAccessionNumber(String accessionNumberString, List<OrderHint> orderHints)  {
 637
 638         DnaSample dnaSample = dao.findByGeneticAccessionNumber(accessionNumberString, null);
 639         DerivedUnitDTO dnaSampleDTO;
 640         if (dnaSample != null){
 641             dnaSampleDTO = new DNASampleDTO(dnaSample);
 642             Collection<SpecimenOrObservationBaseDTO> fieldUnitDTOs = this.findRootUnitDTOs(dnaSampleDTO, new HashMap<>());
 643             // FIXME change return type to Collection<FieldUnitDTO>
 644             if(fieldUnitDTOs.isEmpty()) {
 645                 return null;
 646             } else {
 647                return fieldUnitDTOs.iterator().next();
 648             }
 649         }
 650         return null;
 651     }
 652
 653     @Override
 654     public Pager<SearchResult<SpecimenOrObservationBase>> findByFullText(
 655             Class<? extends SpecimenOrObservationBase> clazz, String queryString, RectangleDTO boundingBox, List<Language> languages,
 656             boolean highlightFragments, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints,
 657             List<String> propertyPaths) throws IOException, LuceneParseException {
 658
 659         LuceneSearch luceneSearch = prepareByFullTextSearch(clazz, queryString, boundingBox, languages, highlightFragments);
 660
 661         // --- execute search
 662         TopGroups<BytesRef> topDocsResultSet;
 663         try {
 664             topDocsResultSet = luceneSearch.executeSearch(pageSize, pageNumber);
 665         } catch (ParseException e) {
 666             LuceneParseException parseException = new LuceneParseException(e.getMessage());
 667             parseException.setStackTrace(e.getStackTrace());
 668             throw parseException;
 669         }
 670
 671         Map<CdmBaseType, String> idFieldMap = new HashMap<>();
 672         idFieldMap.put(CdmBaseType.SPECIMEN_OR_OBSERVATIONBASE, "id");
 673
 674         // --- initialize taxa, highlight matches ....
 675         ISearchResultBuilder searchResultBuilder = new SearchResultBuilder(luceneSearch, luceneSearch.getQuery());
 676         @SuppressWarnings("rawtypes")
 677         List<SearchResult<SpecimenOrObservationBase>> searchResults = searchResultBuilder.createResultSet(
 678                 topDocsResultSet, luceneSearch.getHighlightFields(), dao, idFieldMap, propertyPaths);
 679
 680         int totalHits = topDocsResultSet != null ? topDocsResultSet.totalGroupCount : 0;
 681
 682         return new DefaultPagerImpl<>(pageNumber, Long.valueOf(totalHits), pageSize, searchResults);
 683     }
 684
 685     private LuceneSearch prepareByFullTextSearch(Class<? extends SpecimenOrObservationBase> clazz, String queryString, RectangleDTO bbox,
 686             List<Language> languages, boolean highlightFragments) {
 687
 688         Builder finalQueryBuilder = new Builder();
 689         Builder textQueryBuilder = new Builder();
 690
 691         LuceneSearch luceneSearch = new LuceneSearch(luceneIndexToolProvider, FieldUnit.class);
 692         QueryFactory queryFactory = luceneIndexToolProvider.newQueryFactoryFor(FieldUnit.class);
 693
 694         // --- criteria
 695         luceneSearch.setCdmTypRestriction(clazz);
 696         if (queryString != null) {
 697             textQueryBuilder.add(queryFactory.newTermQuery("titleCache", queryString), Occur.SHOULD);
 698             finalQueryBuilder.add(textQueryBuilder.build(), Occur.MUST);
 699         }
 700
 701         // --- spacial query
 702         if (bbox != null) {
 703             finalQueryBuilder.add(QueryFactory.buildSpatialQueryByRange(bbox, "gatheringEvent.exactLocation.point"), Occur.MUST);
 704         }
 705
 706         luceneSearch.setQuery(finalQueryBuilder.build());
 707
 708         // --- sorting
 709         SortField[] sortFields = new SortField[] { SortField.FIELD_SCORE, new SortField("titleCache__sort", SortField.Type.STRING, false) };
 710         luceneSearch.setSortFields(sortFields);
 711
 712         if (highlightFragments) {
 713             luceneSearch.setHighlightFields(queryFactory.getTextFieldNamesAsArray());
 714         }
 715         return luceneSearch;
 716     }
 717
 718
 719     @Override
 720     @Transactional(readOnly=true)
 721     public Collection<FieldUnit> findFieldUnits(UUID derivedUnitUuid, List<String> propertyPaths) {
 722         //It will search recursively over all {@link DerivationEvent}s and get the "originals" ({@link SpecimenOrObservationBase})
 723         //from which this DerivedUnit was derived until all FieldUnits are found.
 724
 725         // FIXME: use HQL queries to avoid entity instantiation and to increase performance
 726
 727         SpecimenOrObservationBase<?> specimen = load(derivedUnitUuid);
 728         Collection<FieldUnit> fieldUnits = new ArrayList<>();
 729         if (specimen == null){
 730             return null;
 731         }
 732         if (specimen.isInstanceOf(FieldUnit.class)) {
 733             fieldUnits.add(HibernateProxyHelper.deproxy(specimen, FieldUnit.class));
 734         }
 735         else if(specimen.isInstanceOf(DerivedUnit.class)){
 736             fieldUnits.addAll(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class).collectRootUnits(FieldUnit.class));
 737         }
 738
 739         fieldUnits = beanInitializer.initializeAll(fieldUnits, propertyPaths);
 740         return fieldUnits;
 741     }
 742
 743     @Override
 744     @Transactional(readOnly=true)
 745     public Collection<SpecimenOrObservationBase> findRootUnits(UUID derivedUnitUuid, List<String> propertyPaths) {
 746
 747         // FIXME: use HQL queries to avoid entity instantiation and to increase performance
 748
 749         SpecimenOrObservationBase<?> specimen = load(derivedUnitUuid);
 750         Collection<SpecimenOrObservationBase> rootUnits = new ArrayList<>();
 751         if (specimen == null){
 752             return null;
 753         }
 754         if (specimen.isInstanceOf(FieldUnit.class)) {
 755             rootUnits.add(HibernateProxyHelper.deproxy(specimen, FieldUnit.class));
 756         }
 757         else if(specimen.isInstanceOf(DerivedUnit.class)){
 758             rootUnits.addAll(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class).collectRootUnits(SpecimenOrObservationBase.class));
 759         }
 760
 761         rootUnits = beanInitializer.initializeAll(rootUnits, propertyPaths);
 762         return rootUnits;
 763     }
 764
 765     @Override
 766     public Collection<SpecimenOrObservationBaseDTO> findRootUnitDTOs(UUID unitUUID) {
 767
 768
 769         SpecimenOrObservationBase<?> entity = load(unitUUID);
 770         SpecimenOrObservationBaseDTO derivedUnitDTO = SpecimenOrObservationDTOFactory.fromEntity(entity);
 771         Collection<SpecimenOrObservationBaseDTO> rootUnitDTOs = new ArrayList<>();
 772         if(derivedUnitDTO != null) {
 773             if(derivedUnitDTO instanceof FieldUnitDTO) {
 774                 rootUnitDTOs.add(derivedUnitDTO);
 775             } else {
 776                 Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen = new HashMap<>();
 777                 rootUnitDTOs  = findRootUnitDTOs((DerivedUnitDTO)derivedUnitDTO, alreadyCollectedSpecimen);
 778             }
 779         }
 780
 781         return rootUnitDTOs;
 782
 783     }
 784
 785     /**
 786      * Recursively searches all {@link DerivationEvent}s to find all "originals" ({@link SpecimenOrObservationBase})
 787      * from which this DerivedUnit was derived until all FieldUnits are found.
 788      * <p>
 789      * <b>NOTE:</b> The recursive search still is a bit incomplete and may miss originals in the rare case where a
 790      * derivative has more than one original. (see https://dev.e-taxonomy.eu/redmine/issues/9253)
 791      *
 792      * @param derivedUnitDTO
 793      *  The DerivedUnitDTO to start the search from.
 794      * @param alreadyCollectedSpecimen
 795      *  A map to hold all originals that have been sees during the recursive walk.
 796      * @return
 797      *  The collection of all Field Units that are accessible from the derivative from where the search was started.
 798      */
 799     public Collection<SpecimenOrObservationBaseDTO> findRootUnitDTOs(DerivedUnitDTO derivedUnitDTO,
 800             Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 801
 802         HashMap<UUID, SpecimenOrObservationBaseDTO> rootUnitDTOs = new HashMap<>();
 803         _findRootUnitDTOs(derivedUnitDTO, rootUnitDTOs, alreadyCollectedSpecimen);
 804         return rootUnitDTOs.values();
 805
 806     }
 807
 808     /**
 809      * Method for recursive calls, must only be used by {@link #findRootUnitDTOs(DerivedUnitDTO, HashMap)}
 810      * <p>
 811      * It will search recursively over all {@link DerivationEvent}s and get the "originals" ({@link SpecimenOrObservationBase})
 812      * from which this DerivedUnit was derived until all FieldUnits are found.
 813      */
 814     private void _findRootUnitDTOs(DerivedUnitDTO derivedUnitDTO, Map<UUID, SpecimenOrObservationBaseDTO> rootUnitDTOs,
 815                 Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 816
 817         List<String> propertyPaths = new ArrayList<>();
 818
 819         // add the supplied DTO the the alreadyCollectedSpecimen if not yet there
 820         if(!alreadyCollectedSpecimen.containsKey(derivedUnitDTO.getUuid())) {
 821             alreadyCollectedSpecimen.put(derivedUnitDTO.getUuid(), derivedUnitDTO);
 822         }
 823
 824         List<SpecimenOrObservationBase> originals = dao.findOriginalsForDerivedUnit(derivedUnitDTO.getUuid(), propertyPaths);
 825         if (originals.size() > 0){
 826             if (originals.size() > 1){
 827                 logger.warn("The derived unit with uuid " + derivedUnitDTO.getUuid() + "has more than one orginal");
 828             }
 829             // FIXME allow handling multiple originals
 830             SpecimenOrObservationBase<?> original = originals.get(0);
 831             original = HibernateProxyHelper.deproxy(original);
 832
 833             if (alreadyCollectedSpecimen.containsKey(original.getUuid())){
 834                 alreadyCollectedSpecimen.get(original.getUuid()).addDerivative(derivedUnitDTO);
 835     //            if ( alreadyCollectedSpecimen.get(specimen.getUuid()) instanceof FieldUnitDTO){
 836     //                ((FieldUnitDTO)alreadyCollectedSpecimen.get(specimen.getUuid())).getTaxonRelatedDerivedUnits().add(derivedUnitDTO.getUuid());
 837     //            }
 838             }else{
 839                 if(!rootUnitDTOs.containsKey(original.getUuid())){
 840                     // the direct derivatives of the field unit are added in the factory method, so it is guaranteed that
 841                     // the derivedUnitDTO is already contained.
 842                     // ----
 843                     // Don't assemble derivatives for originals since we have them collected already
 844                     // when ascending to the originals, we only want to collect those derivatives which are on the path up to the root
 845                     final Integer maxDepth = 0;
 846                     SpecimenOrObservationBaseDTO originalDTO = SpecimenOrObservationDTOFactory.fromEntity(original, maxDepth);
 847                     originalDTO.addDerivative(derivedUnitDTO);
 848                     alreadyCollectedSpecimen.put(originalDTO.getUuid(), originalDTO);
 849                     if (original instanceof FieldUnit){
 850                         rootUnitDTOs.put(originalDTO.getUuid(), originalDTO);
 851                     }else{
 852                         _findRootUnitDTOs((DerivedUnitDTO) originalDTO, rootUnitDTOs, alreadyCollectedSpecimen);
 853                     }
 854                 } else {
 855                     SpecimenOrObservationBaseDTO previouslyFoundRootUnit = rootUnitDTOs.get(original.getUuid());
 856                     if(!previouslyFoundRootUnit.getDerivatives().stream().anyMatch(uDTO -> uDTO.getUuid().equals(derivedUnitDTO.getUuid()))) {
 857                         previouslyFoundRootUnit.addDerivative(derivedUnitDTO);
 858                     }
 859                 }
 860             }
 861         } else {
 862             rootUnitDTOs.put(derivedUnitDTO.getUuid(), derivedUnitDTO);
 863         }
 864
 865     }
 866
 867     @Override
 868     @Transactional(readOnly=true)
 869     public FieldUnitDTO loadFieldUnitDTO(UUID derivedUnitUuid) {
 870
 871         FieldUnitDTO fieldUnitDTO = null;
 872         DerivedUnitDTO derivedUnitDTO = null;
 873
 874         Map<UUID, SpecimenOrObservationBaseDTO> cycleDetectionMap = new HashMap<>();
 875         SpecimenOrObservationBase<?> derivative = dao.load(derivedUnitUuid);
 876         if(derivative != null){
 877             if (derivative instanceof FieldUnit){
 878                 fieldUnitDTO = FieldUnitDTO.fromEntity((FieldUnit)derivative);
 879                 return fieldUnitDTO;
 880             } else {
 881                 // must be a DerivedUnit otherwise
 882                 derivedUnitDTO = DerivedUnitDTO.fromEntity((DerivedUnit)derivative);
 883                 while(true){
 884
 885                     Set<SpecimenOrObservationBaseDTO> originals = originalDTOs(derivedUnitDTO.getUuid());
 886
 887                     if(originals.isEmpty()){
 888                         break;
 889                     }
 890                     if (originals.size() > 1){
 891                         logger.debug("The derived unit with uuid " + derivedUnitUuid + "has more than one orginal, ignoring all but the first one.");
 892                     }
 893
 894                     SpecimenOrObservationBaseDTO originalDTO = originals.iterator().next();
 895
 896                     // cycle detection and handling
 897                     if(cycleDetectionMap.containsKey(originalDTO.getUuid())){
 898                         // cycle detected!!!
 899                         try {
 900                             throw new Exception();
 901                         } catch(Exception e){
 902                             logger.error("Cycle in derivate graph detected at DerivedUnit with uuid=" + originalDTO.getUuid() , e);
 903                         }
 904                         // to solve the situation for the output we remove the derivate from the more distant graph node
 905                         // by removing it from the derivatives of its original
 906                         // but let the derivate to be added to the original which is closer to the FieldUnit (below at originalDTO.addDerivate(derivedUnitDTO);)
 907                         for(SpecimenOrObservationBaseDTO seenOriginal: cycleDetectionMap.values()){
 908                             for(SpecimenOrObservationBaseDTO derivateDTO : seenOriginal.getDerivatives()){
 909                                 if(derivateDTO.equals(originalDTO)){
 910                                     seenOriginal.getDerivatives().remove(originalDTO);
 911                                 }
 912                             }
 913                         }
 914                     } else {
 915                         cycleDetectionMap.put(originalDTO.getUuid(), originalDTO);
 916                     }
 917
 918                     if (originalDTO instanceof FieldUnitDTO){
 919                         fieldUnitDTO = (FieldUnitDTO)originalDTO;
 920                         break;
 921                     }else{
 922                         // So this must be a DerivedUnitDTO
 923                         if (derivedUnitDTO == null){
 924                             derivedUnitDTO = (DerivedUnitDTO)originalDTO;
 925                         } else {
 926                             derivedUnitDTO = (DerivedUnitDTO)originalDTO;
 927                         }
 928                     }
 929                 }
 930             }
 931         }
 932         return fieldUnitDTO;
 933
 934     }
 935
 936     /**
 937      * @param originalDTO
 938      * @return
 939      */
 940     private Set<SpecimenOrObservationBaseDTO> originalDTOs(UUID derivativeUuid) {
 941
 942         Set<SpecimenOrObservationBaseDTO> dtos = new HashSet<>();
 943         List<SpecimenOrObservationBase> specimens = dao.findOriginalsForDerivedUnit(derivativeUuid, null);
 944         for(SpecimenOrObservationBase sob : specimens){
 945             if(sob instanceof FieldUnit) {
 946                 dtos.add(FieldUnitDTO.fromEntity((FieldUnit)sob));
 947             } else {
 948                 dtos.add(DerivedUnitDTO.fromEntity((DerivedUnit)sob));
 949             }
 950         }
 951         return dtos;
 952     }
 953
 954
 955     @Override
 956     @Transactional(readOnly = false)
 957     public UpdateResult moveSequence(DnaSample from, DnaSample to, Sequence sequence) {
 958         return moveSequence(from.getUuid(), to.getUuid(), sequence.getUuid());
 959     }
 960
 961     @Override
 962     @Transactional(readOnly = false)
 963     public UpdateResult moveSequence(UUID fromUuid, UUID toUuid, UUID sequenceUuid) {
 964         // reload specimens to avoid session conflicts
 965         DnaSample from = (DnaSample) load(fromUuid);
 966         DnaSample to = (DnaSample) load(toUuid);
 967         Sequence sequence = sequenceService.load(sequenceUuid);
 968
 969         if (from == null || to == null || sequence == null) {
 970             throw new TransientObjectException("One of the CDM entities has not been saved to the data base yet. Moving only works for persisted/saved CDM entities.\n" +
 971                     "Operation was move "+sequence+ " from "+from+" to "+to);
 972         }
 973         UpdateResult result = new UpdateResult();
 974         from.removeSequence(sequence);
 975         saveOrUpdate(from);
 976         to.addSequence(sequence);
 977         saveOrUpdate(to);
 978         result.setStatus(Status.OK);
 979         result.addUpdatedObject(from);
 980         result.addUpdatedObject(to);
 981         return result;
 982     }
 983
 984     @Override
 985     @Transactional(readOnly = false)
 986     public boolean moveDerivate(SpecimenOrObservationBase<?> from, SpecimenOrObservationBase<?> to, DerivedUnit derivate) {
 987         return moveDerivate(from!=null?from.getUuid():null, to.getUuid(), derivate.getUuid()).isOk();
 988     }
 989
 990     @Override
 991     @Transactional(readOnly = false)
 992     public UpdateResult moveDerivate(UUID specimenFromUuid, UUID specimenToUuid, UUID derivateUuid) {
 993         // reload specimens to avoid session conflicts
 994         SpecimenOrObservationBase<?> from = null;
 995         if(specimenFromUuid!=null){
 996             from = load(specimenFromUuid);
 997         }
 998         SpecimenOrObservationBase<?> to = load(specimenToUuid);
 999         DerivedUnit derivate = (DerivedUnit) load(derivateUuid);
1000
1001         if ((specimenFromUuid!=null && from == null) || to == null || derivate == null) {
1002             throw new TransientObjectException("One of the CDM entities has not been saved to the data base yet. Moving only works for persisted/saved CDM entities.\n" +
1003                     "Operation was move "+derivate+ " from "+from+" to "+to);
1004         }
1005         UpdateResult result = new UpdateResult();
1006         SpecimenOrObservationType derivateType = derivate.getRecordBasis();
1007         SpecimenOrObservationType toType = to.getRecordBasis();
1008         // check if type is a sub derivate type
1009         if(toType==SpecimenOrObservationType.FieldUnit //moving to FieldUnit always works
1010                 || derivateType==SpecimenOrObservationType.Media //moving media always works
1011                 || (derivateType.isKindOf(toType) && toType!=derivateType)){ //moving only to parent derivate type
1012             if(from!=null){
1013                 // remove derivation event from parent specimen of dragged object
1014                 DerivationEvent eventToRemove = null;
1015                 for (DerivationEvent event : from.getDerivationEvents()) {
1016                     if (event.getDerivatives().contains(derivate)) {
1017                         eventToRemove = event;
1018                         break;
1019                     }
1020                 }
1021                 from.removeDerivationEvent(eventToRemove);
1022                 if(eventToRemove!=null){
1023                     // add new derivation event to target and copy the event parameters of the old one
1024                     DerivationEvent derivedFromNewOriginalEvent = DerivationEvent.NewSimpleInstance(to, derivate, null);
1025                     derivedFromNewOriginalEvent.setActor(eventToRemove.getActor());
1026                     derivedFromNewOriginalEvent.setDescription(eventToRemove.getDescription());
1027                     derivedFromNewOriginalEvent.setInstitution(eventToRemove.getInstitution());
1028                     derivedFromNewOriginalEvent.setTimeperiod(eventToRemove.getTimeperiod());
1029                     derivedFromNewOriginalEvent.setType(eventToRemove.getType());
1030                     to.addDerivationEvent(derivedFromNewOriginalEvent);
1031                     derivate.setDerivedFrom(derivedFromNewOriginalEvent);
1032                 }
1033             }
1034             else{
1035                 //derivative had no parent before so we use empty derivation event
1036                 DerivationEvent derivedFromNewOriginalEvent = DerivationEvent.NewSimpleInstance(to, derivate, null);
1037                 to.addDerivationEvent(derivedFromNewOriginalEvent);
1038                 derivate.setDerivedFrom(derivedFromNewOriginalEvent);
1039             }
1040
1041             if(from!=null){
1042                 saveOrUpdate(from);
1043             }
1044             saveOrUpdate(to);
1045             result.setStatus(Status.OK);
1046             result.addUpdatedObject(from);
1047             result.addUpdatedObject(to);
1048         } else {
1049             result.setStatus(Status.ERROR);
1050         }
1051         return result;
1052     }
1053
1054     @Override
1055     public DeleteResult isDeletable(UUID specimenUuid, DeleteConfiguratorBase config) {
1056         DeleteResult deleteResult = new DeleteResult();
1057         SpecimenOrObservationBase<?> specimen = this.load(specimenUuid);
1058         SpecimenDeleteConfigurator specimenDeleteConfigurator = (SpecimenDeleteConfigurator) config;
1059
1060         // check elements found by super method
1061         Set<CdmBase> relatedObjects = super.isDeletable(specimenUuid, config).getRelatedObjects();
1062         for (CdmBase cdmBase : relatedObjects) {
1063             // check for type designation
1064             if (cdmBase.isInstanceOf(SpecimenTypeDesignation.class) && !specimenDeleteConfigurator.isDeleteFromTypeDesignation()) {
1065                 deleteResult.setAbort();
1066                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is a type specimen."));
1067                 deleteResult.addRelatedObject(cdmBase);
1068                 break;
1069             }
1070             // check for IndividualsAssociations
1071             else if (cdmBase.isInstanceOf(IndividualsAssociation.class) && !specimenDeleteConfigurator.isDeleteFromIndividualsAssociation()) {
1072                 deleteResult.setAbort();
1073                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is still associated via IndividualsAssociations"));
1074                 deleteResult.addRelatedObject(cdmBase);
1075                 break;
1076             }
1077             // check for taxon description
1078             else if(cdmBase.isInstanceOf(TaxonDescription.class)
1079                     && HibernateProxyHelper.deproxy(cdmBase, TaxonDescription.class).getDescribedSpecimenOrObservation().equals(specimen)
1080                     && !specimenDeleteConfigurator.isDeleteFromDescription()){
1081                 deleteResult.setAbort();
1082                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is still used as \"Described Specimen\" in a taxon description."));
1083                 deleteResult.addRelatedObject(cdmBase);
1084                 break;
1085             }
1086             // check for children and parents (derivation events)
1087             else if (cdmBase.isInstanceOf(DerivationEvent.class)) {
1088                 DerivationEvent derivationEvent = HibernateProxyHelper.deproxy(cdmBase, DerivationEvent.class);
1089                 // check if derivation event is empty
1090                 if (!derivationEvent.getDerivatives().isEmpty() && derivationEvent.getOriginals().contains(specimen)) {
1091                     // if derivationEvent is the childEvent and contains derivations
1092 //                    if (derivationEvent.getDerivatives().contains(specimen)) {
1093 //                        //if it is the parent event the specimen is still deletable
1094 //                        continue;
1095 //                    }
1096                     if(!specimenDeleteConfigurator.isDeleteChildren()){
1097                         //if children should not be deleted then it is undeletable
1098                         deleteResult.setAbort();
1099                         deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation still has child derivatives."));
1100                         deleteResult.addRelatedObject(cdmBase);
1101                         break;
1102                     }
1103                     else{
1104                         // check all children if they can be deleted
1105                         Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1106                         DeleteResult childResult = new DeleteResult();
1107                         for (DerivedUnit derivedUnit : derivatives) {
1108                             childResult.includeResult(isDeletable(derivedUnit.getUuid(), specimenDeleteConfigurator));
1109                         }
1110                         if (!childResult.isOk()) {
1111                             deleteResult.setAbort();
1112                             deleteResult.includeResult(childResult);
1113                             deleteResult.addRelatedObject(cdmBase);
1114                             break;
1115                         }
1116                     }
1117                 }
1118             }
1119             // check for amplification
1120             else if (cdmBase.isInstanceOf(AmplificationResult.class)
1121                     && !specimenDeleteConfigurator.isDeleteMolecularData()
1122                     && !specimenDeleteConfigurator.isDeleteChildren()) {
1123                 deleteResult.setAbort();
1124                 deleteResult.addException(new ReferencedObjectUndeletableException("DnaSample is used in amplification results."));
1125                 deleteResult.addRelatedObject(cdmBase);
1126                 break;
1127             }
1128             // check for sequence
1129             else if (cdmBase.isInstanceOf(Sequence.class)
1130                     && !specimenDeleteConfigurator.isDeleteMolecularData()
1131                     && !specimenDeleteConfigurator.isDeleteChildren()) {
1132                 deleteResult.setAbort();
1133                 deleteResult.addException(new ReferencedObjectUndeletableException("DnaSample is used in sequences."));
1134                 deleteResult.addRelatedObject(cdmBase);
1135                 break;
1136             }
1137         }
1138         if (deleteResult.isOk()) {
1139             //add all related object if deletion is OK so they can be handled by the delete() method
1140             deleteResult.addRelatedObjects(relatedObjects);
1141         }
1142         return deleteResult;
1143     }
1144
1145     @Transactional(readOnly = false)
1146     @Override
1147     public DeleteResult delete(UUID specimenUuid, SpecimenDeleteConfigurator config) {
1148         return delete(load(specimenUuid), config);
1149     }
1150
1151     @Transactional(readOnly = false)
1152     @Override
1153     public DeleteResult delete(SpecimenOrObservationBase<?> specimen, SpecimenDeleteConfigurator config) {
1154         specimen = HibernateProxyHelper.deproxy(specimen, SpecimenOrObservationBase.class);
1155
1156         DeleteResult deleteResult = isDeletable(specimen.getUuid(), config);
1157         if (!deleteResult.isOk()) {
1158             return deleteResult;
1159         }
1160
1161         if (config.isDeleteChildren()) {
1162             Set<DerivationEvent> derivationEvents = specimen.getDerivationEvents();
1163             //clone to avoid concurrent modification
1164             //can happen if the child is deleted and deleted its own derivedFrom event
1165             Set<DerivationEvent> derivationEventsClone = new HashSet<>(derivationEvents);
1166             for (DerivationEvent derivationEvent : derivationEventsClone) {
1167                 Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1168                 Iterator<DerivedUnit> it = derivatives.iterator();
1169                 Set<DerivedUnit> derivativesToDelete = new HashSet<>();
1170                 while (it.hasNext()) {
1171                     DerivedUnit unit = it.next();
1172                     derivativesToDelete.add(unit);
1173                 }
1174                 for (DerivedUnit unit: derivativesToDelete){
1175                     DeleteResult derivedDeleteResult = delete(unit, config);
1176                     deleteResult.includeResult(derivedDeleteResult);
1177                 }
1178             }
1179         }
1180
1181         // check related objects
1182         Set<CdmBase> relatedObjects = deleteResult.getRelatedObjects();
1183
1184         for (CdmBase relatedObject : relatedObjects) {
1185             // check for TypeDesignations
1186             if (relatedObject.isInstanceOf(SpecimenTypeDesignation.class)) {
1187                 SpecimenTypeDesignation designation = HibernateProxyHelper.deproxy(relatedObject, SpecimenTypeDesignation.class);
1188                 designation.setTypeSpecimen(null);
1189                 List<TaxonName> typifiedNames = new ArrayList<>();
1190                 typifiedNames.addAll(designation.getTypifiedNames());
1191                 for (TaxonName taxonName : typifiedNames) {
1192                     taxonName.removeTypeDesignation(designation);
1193                 }
1194             }
1195             // delete IndividualsAssociation
1196             if (relatedObject.isInstanceOf(IndividualsAssociation.class)) {
1197                 IndividualsAssociation association = HibernateProxyHelper.deproxy(relatedObject, IndividualsAssociation.class);
1198                 association.setAssociatedSpecimenOrObservation(null);
1199                 association.getInDescription().removeElement(association);
1200             }
1201             // check for "described specimen" (deprecated)
1202             if (relatedObject.isInstanceOf(TaxonDescription.class)) {
1203                 TaxonDescription description = HibernateProxyHelper.deproxy(relatedObject, TaxonDescription.class);
1204                 description.setDescribedSpecimenOrObservation(null);
1205             }
1206             // check for specimen description
1207             if (relatedObject.isInstanceOf(SpecimenDescription.class)) {
1208                 SpecimenDescription specimenDescription = HibernateProxyHelper.deproxy(relatedObject, SpecimenDescription.class);
1209                 specimenDescription.setDescribedSpecimenOrObservation(null);
1210                 // check if description is a description of the given specimen
1211                 if (specimen.getDescriptions().contains(specimenDescription)) {
1212                     specimen.removeDescription(specimenDescription);
1213                 }
1214                 DeleteResult descriptionDelete = descriptionService.isDeletable(specimenDescription.getUuid(), null);
1215                 if (descriptionDelete.isOk()){
1216                     deleteResult.includeResult(descriptionService.delete(specimenDescription));
1217                 }
1218             }
1219             // check for amplification
1220             if (relatedObject.isInstanceOf(AmplificationResult.class)) {
1221                 AmplificationResult amplificationResult = HibernateProxyHelper.deproxy(relatedObject, AmplificationResult.class);
1222                 amplificationResult.getDnaSample().removeAmplificationResult(amplificationResult);
1223             }
1224             // check for sequence
1225             if (relatedObject.isInstanceOf(Sequence.class)) {
1226                 Sequence sequence = HibernateProxyHelper.deproxy(relatedObject, Sequence.class);
1227                 sequence.getDnaSample().removeSequence(sequence);
1228             }
1229             // check for children and parents (derivation events)
1230             if (relatedObject.isInstanceOf(DerivationEvent.class)) {
1231                 DerivationEvent derivationEvent = HibernateProxyHelper.deproxy(relatedObject, DerivationEvent.class);
1232                 // parent derivation event (derivedFrom)
1233                 if (derivationEvent.getDerivatives().contains(specimen) && specimen.isInstanceOf(DerivedUnit.class)) {
1234                     derivationEvent.removeDerivative(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class));
1235                     if (derivationEvent.getDerivatives().isEmpty()) {
1236                         Set<SpecimenOrObservationBase> originals = new HashSet<>();
1237                         originals.addAll(derivationEvent.getOriginals());
1238                         for (SpecimenOrObservationBase specimenOrObservationBase : originals) {
1239                             specimenOrObservationBase.removeDerivationEvent(derivationEvent);
1240                             deleteResult.addUpdatedObject(specimenOrObservationBase);
1241                         }
1242                         // if derivationEvent has no derivates anymore, delete it
1243                         deleteResult.includeResult(eventService.delete(derivationEvent));
1244                     }
1245                 }
1246                 else{
1247                     //child derivation events should not occur since we delete the hierarchy from bottom to top
1248                 }
1249             }
1250         }
1251         if (specimen instanceof FieldUnit){
1252             FieldUnit fieldUnit = HibernateProxyHelper.deproxy(specimen, FieldUnit.class);
1253             GatheringEvent event = fieldUnit.getGatheringEvent();
1254             fieldUnit.setGatheringEvent(null);
1255             if (event != null){
1256                 DeleteResult result = eventService.isDeletable(event.getUuid(), null);
1257                 if (result.isOk()){
1258                     deleteResult.includeResult( eventService.delete(event));
1259                 }
1260             }
1261
1262         }
1263         deleteResult.includeResult(delete(specimen));
1264
1265         return deleteResult;
1266     }
1267
1268     @Override
1269     public Collection<IndividualsAssociation> listIndividualsAssociations(SpecimenOrObservationBase<?> specimen, Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1270         return dao.listIndividualsAssociations(specimen, limit, start, orderHints, propertyPaths);
1271     }
1272
1273     /**
1274      * {@inheritDoc}
1275      */
1276     @Override
1277     public Collection<TaxonBase<?>> listAssociatedTaxa(SpecimenOrObservationBase<?> specimen, Integer limit,
1278             Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1279         return listAssociatedTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1280     }
1281     @Override
1282     public Collection<TaxonBase<?>> listAssociatedTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished,
1283             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1284         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1285
1286         //individuals associations
1287         associatedTaxa.addAll(listIndividualsAssociationTaxa(specimen, includeUnpublished, limit, start, orderHints, propertyPaths));
1288         //type designation
1289         if(specimen.isInstanceOf(DerivedUnit.class)){
1290             associatedTaxa.addAll(listTypeDesignationTaxa(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class),
1291                   includeUnpublished, limit, start, orderHints, propertyPaths));
1292         }
1293         //determinations
1294         associatedTaxa.addAll(listDeterminedTaxa(specimen, includeUnpublished, limit, start, orderHints, propertyPaths));
1295
1296         return associatedTaxa;
1297     }
1298
1299
1300
1301     /**
1302      * {@inheritDoc}
1303      */
1304     @Override
1305     public Collection<TaxonBase<?>> listDeterminedTaxa(SpecimenOrObservationBase<?> specimen, Integer limit,
1306             Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1307         return listDeterminedTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1308     }
1309     @Override
1310     public Collection<TaxonBase<?>> listDeterminedTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished, Integer limit, Integer start,
1311             List<OrderHint> orderHints, List<String> propertyPaths) {
1312         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1313         for (DeterminationEvent determinationEvent : listDeterminationEvents(specimen, limit, start, orderHints, propertyPaths)) {
1314             if(determinationEvent.getIdentifiedUnit().equals(specimen)){
1315                 if(determinationEvent.getTaxon()!=null){
1316                     associatedTaxa.add(taxonService.load(determinationEvent.getTaxon().getUuid(), includeUnpublished, propertyPaths));
1317                 }
1318                 if(determinationEvent.getTaxonName()!=null){
1319                     Collection<TaxonBase> taxonBases = determinationEvent.getTaxonName().getTaxonBases();
1320                     for (TaxonBase taxonBase : taxonBases) {
1321                         associatedTaxa.add(taxonService.load(taxonBase.getUuid(), includeUnpublished, propertyPaths));
1322                     }
1323                 }
1324             }
1325         }
1326         return associatedTaxa;
1327     }
1328
1329     /**
1330      * {@inheritDoc}
1331      */
1332     @Override
1333     public Collection<TaxonBase<?>> listTypeDesignationTaxa(DerivedUnit specimen, Integer limit, Integer start,
1334             List<OrderHint> orderHints, List<String> propertyPaths) {
1335         return listTypeDesignationTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1336     }
1337     @Override
1338     public Collection<TaxonBase<?>> listTypeDesignationTaxa(DerivedUnit specimen, boolean includeUnpublished,
1339             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1340         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1341         for (SpecimenTypeDesignation typeDesignation : listTypeDesignations(specimen, limit, start, orderHints, propertyPaths)) {
1342             if(typeDesignation.getTypeSpecimen().equals(specimen)){
1343                 Set<TaxonName> typifiedNames = typeDesignation.getTypifiedNames();
1344                 for (TaxonName taxonName : typifiedNames) {
1345                     Set<Taxon> taxa = taxonName.getTaxa();
1346                     for (Taxon taxon : taxa) {
1347                         associatedTaxa.add(taxonService.load(taxon.getUuid(), includeUnpublished, propertyPaths));
1348                     }
1349                 }
1350             }
1351         }
1352         return associatedTaxa;
1353     }
1354
1355     /**
1356      * {@inheritDoc}
1357      */
1358     @Override
1359     public Collection<TaxonBase<?>> listIndividualsAssociationTaxa(SpecimenOrObservationBase<?> specimen, Integer limit,
1360             Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1361         return listIndividualsAssociationTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1362     }
1363
1364     @Override
1365     public Collection<TaxonBase<?>> listIndividualsAssociationTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished,
1366             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1367         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1368         for (IndividualsAssociation individualsAssociation : listIndividualsAssociations(specimen, null, null, null, null)) {
1369             if(individualsAssociation.getInDescription().isInstanceOf(TaxonDescription.class)){
1370                 TaxonDescription taxonDescription = HibernateProxyHelper.deproxy(individualsAssociation.getInDescription(), TaxonDescription.class);
1371                 if(taxonDescription.getTaxon()!=null){
1372                     associatedTaxa.add(taxonService.load(taxonDescription.getTaxon().getUuid(), includeUnpublished, propertyPaths));
1373                 }
1374             }
1375         }
1376         return associatedTaxa;
1377     }
1378
1379     @Override
1380     public Collection<DeterminationEvent> listDeterminationEvents(SpecimenOrObservationBase<?> specimen,
1381             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1382         return dao.listDeterminationEvents(specimen, limit, start, orderHints, propertyPaths);
1383     }
1384
1385     @Override
1386     public Map<DerivedUnit, Collection<SpecimenTypeDesignation>> listTypeDesignations(
1387             Collection<DerivedUnit> specimens, Integer limit, Integer start,
1388             List<OrderHint> orderHints, List<String> propertyPaths) {
1389         Map<DerivedUnit, Collection<SpecimenTypeDesignation>> typeDesignationMap = new HashMap<>();
1390         for (DerivedUnit specimen : specimens) {
1391             Collection<SpecimenTypeDesignation> typeDesignations = listTypeDesignations(specimen, limit, start, orderHints, propertyPaths);
1392             typeDesignationMap.put(specimen, typeDesignations);
1393         }
1394         return typeDesignationMap;
1395     }
1396
1397     @Override
1398     public Collection<SpecimenTypeDesignation> listTypeDesignations(DerivedUnit specimen,
1399             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1400         return dao.listTypeDesignations(specimen, limit, start, orderHints, propertyPaths);
1401     }
1402
1403     @Override
1404     public Collection<DescriptionBase<?>> listDescriptionsWithDescriptionSpecimen(
1405             SpecimenOrObservationBase<?> specimen, Integer limit, Integer start, List<OrderHint> orderHints,
1406             List<String> propertyPaths) {
1407         return dao.listDescriptionsWithDescriptionSpecimen(specimen, limit, start, orderHints, propertyPaths);
1408     }
1409
1410     @Override
1411     public Collection<DescriptionElementBase> getCharacterDataForSpecimen(UUID specimenUuid) {
1412         SpecimenOrObservationBase<?> specimen = load(specimenUuid);
1413         if (specimen != null) {
1414             return specimen.characterData();
1415         }
1416         else{
1417             throw new DataRetrievalFailureException("Specimen with the given uuid not found in the data base");
1418         }
1419     }
1420
1421     @Override
1422     public long countByTitle(IIdentifiableEntityServiceConfigurator<SpecimenOrObservationBase> config){
1423         if (config instanceof FindOccurrencesConfigurator) {
1424             FindOccurrencesConfigurator occurrenceConfig = (FindOccurrencesConfigurator) config;
1425             Taxon taxon = null;
1426             if(occurrenceConfig.getAssociatedTaxonUuid()!=null){
1427                 TaxonBase<?> taxonBase = taxonService.load(occurrenceConfig.getAssociatedTaxonUuid());
1428                 if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1429                     taxon = HibernateProxyHelper.deproxy(taxonBase, Taxon.class);
1430                 }
1431             }
1432             TaxonName taxonName = null;
1433             if(occurrenceConfig.getAssociatedTaxonNameUuid()!=null){
1434                 taxonName = nameService.load(occurrenceConfig.getAssociatedTaxonNameUuid());
1435             }
1436             /*TODO: #6484 Neither isRetrieveIndirectlyAssociatedSpecimens() nor the AssignmentStatus
1437              * is currently reflected in the HQL query. So using these in the count method will
1438              * significantly slow down this method as we have to retrieve the entities instead of
1439              * just the amount.
1440              */
1441             if(occurrenceConfig.isRetrieveIndirectlyAssociatedSpecimens() || !occurrenceConfig.getAssignmentStatus().equals(AssignmentStatus.ALL_SPECIMENS)){
1442                 List<SpecimenOrObservationBase> occurrences = new ArrayList<>();
1443                 List<SpecimenOrObservationBase> sobs = dao.findOccurrences(occurrenceConfig.getClazz(),
1444                         occurrenceConfig.getTitleSearchStringSqlized(), occurrenceConfig.getSignificantIdentifier(),
1445                         occurrenceConfig.getSpecimenType(), taxon, taxonName, occurrenceConfig.getMatchMode(), null, null,
1446                         occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1447                 occurrences.addAll(sobs);
1448                 occurrences = filterOccurencesByAssignmentAndHierarchy(occurrenceConfig, occurrences, taxon, taxonName);
1449                 return occurrences.size();
1450             }
1451
1452             return dao.countOccurrences(occurrenceConfig.getClazz(),
1453                     occurrenceConfig.getTitleSearchStringSqlized(), occurrenceConfig.getSignificantIdentifier(),
1454                     occurrenceConfig.getSpecimenType(), taxon, taxonName, occurrenceConfig.getMatchMode(), null, null,
1455                     occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1456         }
1457         else{
1458             return super.countByTitle(config);
1459         }
1460     }
1461
1462     @Override
1463     public Pager<UuidAndTitleCache<SpecimenOrObservationBase>> findByTitleUuidAndTitleCache(
1464             FindOccurrencesConfigurator config){
1465         List<UuidAndTitleCache<SpecimenOrObservationBase>> occurrences = new ArrayList<>();
1466         Taxon taxon = null;
1467         if(config.getAssociatedTaxonUuid()!=null){
1468             TaxonBase<?> taxonBase = taxonService.load(config.getAssociatedTaxonUuid());
1469             if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1470                 taxon = CdmBase.deproxy(taxonBase, Taxon.class);
1471             }
1472         }
1473         TaxonName taxonName = null;
1474         if(config.getAssociatedTaxonNameUuid()!=null){
1475             taxonName = nameService.load(config.getAssociatedTaxonNameUuid());
1476         }
1477         occurrences.addAll(dao.findOccurrencesUuidAndTitleCache(config.getClazz(),
1478                 config.getTitleSearchString(), config.getSignificantIdentifier(),
1479                 config.getSpecimenType(), taxon, taxonName, config.getMatchMode(), null, null,
1480                 config.getOrderHints()));
1481         long count = Integer.valueOf(occurrences.size()).longValue();
1482         return new DefaultPagerImpl<>(config.getPageNumber(), count, config.getPageSize(), occurrences);
1483     }
1484
1485     @Override
1486     public List<DerivedUnitDTO> findByTitleDerivedUnitDTO(FindOccurrencesConfigurator config) {
1487         Taxon taxon = null;
1488         if(config.getAssociatedTaxonUuid()!=null){
1489             TaxonBase<?> taxonBase = taxonService.load(config.getAssociatedTaxonUuid());
1490             if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1491                 taxon = CdmBase.deproxy(taxonBase, Taxon.class);
1492             }
1493         }
1494         TaxonName taxonName = null;
1495         if(config.getAssociatedTaxonNameUuid()!=null){
1496             taxonName = nameService.load(config.getAssociatedTaxonNameUuid());
1497         }
1498         List<DerivedUnit> occurrences = new ArrayList<>();
1499         occurrences.addAll(dao.findOccurrences(DerivedUnit.class,
1500                 config.getTitleSearchString(), config.getSignificantIdentifier(),
1501                 config.getSpecimenType(), taxon, taxonName, config.getMatchMode(), null, null,
1502                 config.getOrderHints(), null));
1503
1504         List<DerivedUnitDTO> dtos = new ArrayList<>();
1505         occurrences.forEach(derivedUnit->dtos.add(assembleDerivedUnitDTO(derivedUnit)));
1506         return dtos;
1507     }
1508
1509     @Override
1510     public <S extends SpecimenOrObservationBase> Pager<S> findByTitle(
1511             IIdentifiableEntityServiceConfigurator<S> config) {
1512         if (config instanceof FindOccurrencesConfigurator) {
1513             FindOccurrencesConfigurator occurrenceConfig = (FindOccurrencesConfigurator) config;
1514             List<SpecimenOrObservationBase> occurrences = new ArrayList<>();
1515             Taxon taxon = null;
1516             if(occurrenceConfig.getAssociatedTaxonUuid()!=null){
1517                 TaxonBase<?> taxonBase = taxonService.load(occurrenceConfig.getAssociatedTaxonUuid());
1518                 if(taxonBase.isInstanceOf(Taxon.class)){
1519                     taxon = HibernateProxyHelper.deproxy(taxonBase, Taxon.class);
1520                 }
1521             }
1522             TaxonName taxonName = null;
1523             if(occurrenceConfig.getAssociatedTaxonNameUuid()!=null){
1524                 taxonName = nameService.load(occurrenceConfig.getAssociatedTaxonNameUuid());
1525             }
1526             List<? extends SpecimenOrObservationBase> foundOccurrences = dao.findOccurrences(occurrenceConfig.getClazz(),
1527                     occurrenceConfig.getTitleSearchString(), occurrenceConfig.getSignificantIdentifier(),
1528                     occurrenceConfig.getSpecimenType(), taxon, taxonName, occurrenceConfig.getMatchMode(), null, null,
1529                     occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1530             occurrences.addAll(foundOccurrences);
1531             occurrences = filterOccurencesByAssignmentAndHierarchy(occurrenceConfig, occurrences, taxon, taxonName);
1532
1533             long count = Integer.valueOf(occurrences.size()).longValue();
1534             return new DefaultPagerImpl<>(config.getPageNumber(), count, config.getPageSize(), (List<S>)occurrences);
1535         }
1536         return super.findByTitle(config);
1537     }
1538
1539     private List<SpecimenOrObservationBase> filterOccurencesByAssignmentAndHierarchy(
1540             FindOccurrencesConfigurator occurrenceConfig, List<SpecimenOrObservationBase> occurrences, Taxon taxon,
1541             TaxonName taxonName) {
1542         //filter out (un-)assigned specimens
1543         if(taxon==null && taxonName==null){
1544             AssignmentStatus assignmentStatus = occurrenceConfig.getAssignmentStatus();
1545             List<SpecimenOrObservationBase> specimenWithAssociations = new ArrayList<>();
1546             if(!assignmentStatus.equals(AssignmentStatus.ALL_SPECIMENS)){
1547                 for (SpecimenOrObservationBase<?> specimenOrObservationBase : occurrences) {
1548                     boolean includeUnpublished = true;  //TODO not sure if this is correct, maybe we have to propagate publish flag to higher methods.
1549                     Collection<TaxonBase<?>> associatedTaxa = listAssociatedTaxa(specimenOrObservationBase,
1550                             includeUnpublished, null, null, null, null);
1551                     if(!associatedTaxa.isEmpty()){
1552                         specimenWithAssociations.add(specimenOrObservationBase);
1553                     }
1554                 }
1555             }
1556             if(assignmentStatus.equals(AssignmentStatus.UNASSIGNED_SPECIMENS)){
1557                 occurrences.removeAll(specimenWithAssociations);
1558             }
1559             if(assignmentStatus.equals(AssignmentStatus.ASSIGNED_SPECIMENS)){
1560                 occurrences = new ArrayList<>(specimenWithAssociations);
1561             }
1562         }
1563         // indirectly associated specimens
1564         if(occurrenceConfig.isRetrieveIndirectlyAssociatedSpecimens()){
1565             List<SpecimenOrObservationBase> indirectlyAssociatedOccurrences = new ArrayList<>(occurrences);
1566             for (SpecimenOrObservationBase<?> specimen : occurrences) {
1567                 List<SpecimenOrObservationBase<?>> allHierarchyDerivates = getAllHierarchyDerivatives(specimen);
1568                 for (SpecimenOrObservationBase<?> specimenOrObservationBase : allHierarchyDerivates) {
1569                     if(!occurrences.contains(specimenOrObservationBase)){
1570                         indirectlyAssociatedOccurrences.add(specimenOrObservationBase);
1571                     }
1572                 }
1573             }
1574             occurrences = indirectlyAssociatedOccurrences;
1575         }
1576         return occurrences;
1577     }
1578
1579     @Override
1580     public List<SpecimenOrObservationBase<?>> getAllHierarchyDerivatives(SpecimenOrObservationBase<?> specimen){
1581         List<SpecimenOrObservationBase<?>> allHierarchyDerivatives = new ArrayList<>();
1582         Collection<FieldUnit> fieldUnits = findFieldUnits(specimen.getUuid(), null);
1583         if(fieldUnits.isEmpty()){
1584             allHierarchyDerivatives.add(specimen);
1585             allHierarchyDerivatives.addAll(getAllChildDerivatives(specimen));
1586         }
1587         else{
1588             for (FieldUnit fieldUnit : fieldUnits) {
1589                 allHierarchyDerivatives.add(fieldUnit);
1590                 allHierarchyDerivatives.addAll(getAllChildDerivatives(fieldUnit));
1591             }
1592         }
1593         return allHierarchyDerivatives;
1594     }
1595
1596     @Override
1597     public List<DerivedUnit> getAllChildDerivatives(UUID specimenUuid){
1598         return getAllChildDerivatives(load(specimenUuid));
1599     }
1600
1601     @Override
1602     public List<DerivedUnit> getAllChildDerivatives(SpecimenOrObservationBase<?> specimen){
1603         if (specimen == null){
1604             return null;
1605         }
1606         List<DerivedUnit> childDerivate = new ArrayList<>();
1607         Set<DerivationEvent> derivationEvents = specimen.getDerivationEvents();
1608         for (DerivationEvent derivationEvent : derivationEvents) {
1609             Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1610             for (DerivedUnit derivedUnit : derivatives) {
1611                 childDerivate.add(derivedUnit);
1612                 childDerivate.addAll(getAllChildDerivatives(derivedUnit.getUuid()));
1613             }
1614         }
1615         return childDerivate;
1616     }
1617
1618     @Override
1619     public long countOccurrences(IIdentifiableEntityServiceConfigurator<SpecimenOrObservationBase> config){
1620         return countByTitle(config);
1621     }
1622
1623     @Override
1624     public List<FieldUnit> findFieldUnitsForGatheringEvent(UUID gatheringEventUuid) {
1625         return dao.findFieldUnitsForGatheringEvent(gatheringEventUuid, null, null, null, null);
1626     }
1627
1628     @Override
1629     public List<Point> findPointsForFieldUnitList(List<UUID> fieldUnitUuids) {
1630         return dao.findPointsForFieldUnitList(fieldUnitUuids);
1631     }
1632 }