cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/OccurrenceServiceImpl.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9 package eu.etaxonomy.cdm.api.service;
  10
  11 import java.io.IOException;
  12 import java.util.ArrayList;
  13 import java.util.Collection;
  14 import java.util.Collections;
  15 import java.util.Comparator;
  16 import java.util.EnumSet;
  17 import java.util.HashMap;
  18 import java.util.HashSet;
  19 import java.util.Iterator;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.UUID;
  24 import java.util.stream.Collectors;
  25
  26 import org.apache.commons.lang3.StringUtils;
  27 import org.apache.logging.log4j.LogManager;
  28 import org.apache.logging.log4j.Logger;
  29 import org.apache.lucene.queryparser.classic.ParseException;
  30 import org.apache.lucene.search.BooleanClause.Occur;
  31 import org.apache.lucene.search.BooleanQuery.Builder;
  32 import org.apache.lucene.search.SortField;
  33 import org.apache.lucene.search.grouping.TopGroups;
  34 import org.apache.lucene.util.BytesRef;
  35 import org.hibernate.TransientObjectException;
  36 import org.springframework.beans.factory.annotation.Autowired;
  37 import org.springframework.dao.DataRetrievalFailureException;
  38 import org.springframework.stereotype.Service;
  39 import org.springframework.transaction.annotation.Transactional;
  40
  41 import eu.etaxonomy.cdm.api.service.UpdateResult.Status;
  42 import eu.etaxonomy.cdm.api.service.config.DeleteConfiguratorBase;
  43 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  44 import eu.etaxonomy.cdm.api.service.config.IIdentifiableEntityServiceConfigurator;
  45 import eu.etaxonomy.cdm.api.service.config.SpecimenDeleteConfigurator;
  46 import eu.etaxonomy.cdm.api.service.dto.DNASampleDTO;
  47 import eu.etaxonomy.cdm.api.service.dto.DerivedUnitDTO;
  48 import eu.etaxonomy.cdm.api.service.dto.FieldUnitDTO;
  49 import eu.etaxonomy.cdm.api.service.dto.MediaDTO;
  50 import eu.etaxonomy.cdm.api.service.dto.RectangleDTO;
  51 import eu.etaxonomy.cdm.api.service.dto.SpecimenOrObservationBaseDTO;
  52 import eu.etaxonomy.cdm.api.service.dto.SpecimenOrObservationDTOFactory;
  53 import eu.etaxonomy.cdm.api.service.exception.ReferencedObjectUndeletableException;
  54 import eu.etaxonomy.cdm.api.service.molecular.ISequenceService;
  55 import eu.etaxonomy.cdm.api.service.pager.Pager;
  56 import eu.etaxonomy.cdm.api.service.pager.impl.AbstractPagerImpl;
  57 import eu.etaxonomy.cdm.api.service.pager.impl.DefaultPagerImpl;
  58 import eu.etaxonomy.cdm.api.service.search.ILuceneIndexToolProvider;
  59 import eu.etaxonomy.cdm.api.service.search.ISearchResultBuilder;
  60 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  61 import eu.etaxonomy.cdm.api.service.search.LuceneSearch;
  62 import eu.etaxonomy.cdm.api.service.search.QueryFactory;
  63 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  64 import eu.etaxonomy.cdm.api.service.search.SearchResultBuilder;
  65 import eu.etaxonomy.cdm.api.util.TaxonRelationshipEdge;
  66 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  67 import eu.etaxonomy.cdm.compare.common.PartialComparator;
  68 import eu.etaxonomy.cdm.facade.DerivedUnitFacade;
  69 import eu.etaxonomy.cdm.facade.DerivedUnitFacadeConfigurator;
  70 import eu.etaxonomy.cdm.facade.DerivedUnitFacadeNotSupportedException;
  71 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  72 import eu.etaxonomy.cdm.model.CdmBaseType;
  73 import eu.etaxonomy.cdm.model.common.CdmBase;
  74 import eu.etaxonomy.cdm.model.common.Language;
  75 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  76 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  77 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  78 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  79 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  80 import eu.etaxonomy.cdm.model.location.Point;
  81 import eu.etaxonomy.cdm.model.media.Media;
  82 import eu.etaxonomy.cdm.model.molecular.AmplificationResult;
  83 import eu.etaxonomy.cdm.model.molecular.DnaSample;
  84 import eu.etaxonomy.cdm.model.molecular.Sequence;
  85 import eu.etaxonomy.cdm.model.molecular.SingleRead;
  86 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  87 import eu.etaxonomy.cdm.model.name.TaxonName;
  88 import eu.etaxonomy.cdm.model.name.TypeDesignationBase;
  89 import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
  90 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  91 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  92 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  93 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  94 import eu.etaxonomy.cdm.model.occurrence.MediaSpecimen;
  95 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  96 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  97 import eu.etaxonomy.cdm.model.taxon.Taxon;
  98 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  99 import eu.etaxonomy.cdm.persistence.dao.initializer.AbstractBeanInitializer;
 100 import eu.etaxonomy.cdm.persistence.dao.occurrence.IOccurrenceDao;
 101 import eu.etaxonomy.cdm.persistence.dto.SpecimenNodeWrapper;
 102 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
 103 import eu.etaxonomy.cdm.persistence.query.AssignmentStatus;
 104 import eu.etaxonomy.cdm.persistence.query.OrderHint;
 105 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
 106
 107 /**
 108  * @author a.babadshanjan
 109  * @since 01.09.2008
 110  */
 111 @Service
 112 @Transactional(readOnly = true)
 113 public class OccurrenceServiceImpl
 114         extends IdentifiableServiceBase<SpecimenOrObservationBase, IOccurrenceDao>
 115         implements IOccurrenceService {
 116
 117     private static final Logger logger = LogManager.getLogger();
 118
 119     @Autowired
 120     private IDescriptionService descriptionService;
 121
 122     @Autowired
 123     private INameService nameService;
 124
 125     @Autowired
 126     private IEventBaseService eventService;
 127
 128     @Autowired
 129     private ITaxonService taxonService;
 130
 131     @Autowired
 132     private ISequenceService sequenceService;
 133
 134     @Autowired
 135     private AbstractBeanInitializer<?> beanInitializer;
 136
 137     @Autowired
 138     private ILuceneIndexToolProvider luceneIndexToolProvider;
 139
 140     public OccurrenceServiceImpl() {
 141         logger.debug("Load OccurrenceService Bean");
 142     }
 143
 144     @Override
 145     @Transactional(readOnly = false)
 146     public UpdateResult updateCaches(Class<? extends SpecimenOrObservationBase> clazz, Integer stepSize, IIdentifiableEntityCacheStrategy<SpecimenOrObservationBase> cacheStrategy, IProgressMonitor monitor) {
 147         if (clazz == null) {
 148             clazz = SpecimenOrObservationBase.class;
 149         }
 150         return super.updateCachesImpl(clazz, stepSize, cacheStrategy, monitor);
 151     }
 152
 153     @Override
 154     @Autowired
 155     protected void setDao(IOccurrenceDao dao) {
 156         this.dao = dao;
 157     }
 158
 159     @Override
 160     public Pager<DerivationEvent> getDerivationEvents(SpecimenOrObservationBase occurence, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 161         long numberOfResults = dao.countDerivationEvents(occurence);
 162
 163         List<DerivationEvent> results = new ArrayList<>();
 164         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 165             results = dao.getDerivationEvents(occurence, pageSize, pageNumber, propertyPaths);
 166         }
 167
 168         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 169     }
 170
 171     @Override
 172     public long countDeterminations(SpecimenOrObservationBase occurence, TaxonBase taxonbase) {
 173         return dao.countDeterminations(occurence, taxonbase);
 174     }
 175
 176     @Override
 177     public Pager<DeterminationEvent> getDeterminations(SpecimenOrObservationBase occurrence, TaxonBase taxonBase,
 178             Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 179         long numberOfResults = dao.countDeterminations(occurrence, taxonBase);
 180
 181         List<DeterminationEvent> results = new ArrayList<>();
 182         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 183             results = dao.getDeterminations(occurrence, taxonBase, pageSize, pageNumber, propertyPaths);
 184         }
 185
 186         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 187     }
 188
 189     @Override
 190     public Pager<Media> getMedia(SpecimenOrObservationBase occurence, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
 191         long numberOfResults = dao.countMedia(occurence);
 192
 193         List<Media> results = new ArrayList<>();
 194         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 195             results = dao.getMedia(occurence, pageSize, pageNumber, propertyPaths);
 196         }
 197
 198         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 199     }
 200
 201     @Override
 202     public Pager<MediaDTO> getMediaDTOs(SpecimenOrObservationBase<?> occurence, Integer pageSize, Integer pageNumber) {
 203         long numberOfResults = dao.countMedia(occurence);
 204
 205         List<Media> results = new ArrayList<>();
 206         if(AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)) {
 207             results = dao.getMedia(occurence, pageSize, pageNumber, null);
 208         }
 209         List<MediaDTO> mediaDTOs = results.stream()
 210                 .map(m -> MediaDTO.fromEntity(m))
 211                 .flatMap(dtos -> dtos.stream())
 212                 .collect(Collectors.toList()
 213                 );
 214         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, mediaDTOs);
 215     }
 216     @Override
 217     public Pager<Media> getMediaInHierarchy(SpecimenOrObservationBase<?> rootOccurence, boolean collectOriginalMedia, boolean collectDerivateMedia, Integer pageSize,
 218             Integer pageNumber, List<String> propertyPaths) {
 219          List<Media> media = new ArrayList<>();
 220          //media specimens
 221          if(rootOccurence.isInstanceOf(MediaSpecimen.class)){
 222              MediaSpecimen mediaSpecimen = HibernateProxyHelper.deproxy(rootOccurence, MediaSpecimen.class);
 223              media.add(mediaSpecimen.getMediaSpecimen());
 224          }
 225          // pherograms & gelPhotos
 226          if (rootOccurence.isInstanceOf(DnaSample.class)) {
 227              DnaSample dnaSample = CdmBase.deproxy(rootOccurence, DnaSample.class);
 228              Set<Sequence> sequences = dnaSample.getSequences();
 229              //we do show only those gelPhotos which lead to a consensus sequence
 230              for (Sequence sequence : sequences) {
 231                  Set<Media> dnaRelatedMedia = new HashSet<>();
 232                  for (SingleRead singleRead : sequence.getSingleReads()){
 233                      AmplificationResult amplification = singleRead.getAmplificationResult();
 234                      dnaRelatedMedia.add(amplification.getGelPhoto());
 235                      dnaRelatedMedia.add(singleRead.getPherogram());
 236                      dnaRelatedMedia.remove(null);
 237                  }
 238                  media.addAll(dnaRelatedMedia);
 239              }
 240          }
 241          if(rootOccurence.isInstanceOf(DerivedUnit.class)){
 242              DerivedUnit derivedUnit = HibernateProxyHelper.deproxy(rootOccurence, DerivedUnit.class);
 243              if (collectDerivateMedia) {
 244                      for (DerivationEvent derivationEvent : derivedUnit.getDerivationEvents()) {
 245                          for (DerivedUnit childDerivative : derivationEvent.getDerivatives()) {
 246                                  //collectOriginalMedia should only called for the first derived unit
 247                              media.addAll(getMediaInHierarchy(childDerivative, false, true, pageSize, pageNumber, propertyPaths).getRecords());
 248                          }
 249                      }
 250              }
 251              if (collectOriginalMedia) {
 252                  for (SpecimenOrObservationBase original : derivedUnit.getOriginals()) {
 253                                  //collect media to the top of the tree
 254                              media.addAll(getMediaInHierarchy(original, true, false, pageSize, pageNumber, propertyPaths).getRecords());
 255                      }
 256                  }
 257          }
 258
 259
 260          return new DefaultPagerImpl<>(pageNumber, Long.valueOf(media.size()), pageSize, media);
 261     }
 262
 263
 264     @Override
 265     public Pager<Media> getMediaInHierarchy(SpecimenOrObservationBase<?> rootOccurence, Integer pageSize,
 266             Integer pageNumber, List<String> propertyPaths) {
 267         return getMediaInHierarchy(rootOccurence, false, true, pageSize,
 268                 pageNumber, propertyPaths);
 269
 270     }
 271
 272     @Override
 273     public Pager<SpecimenOrObservationBase> list(Class<? extends SpecimenOrObservationBase> type, TaxonName determinedAs,
 274             Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 275         long numberOfResults = dao.count(type, determinedAs);
 276         @SuppressWarnings("rawtypes")
 277         List<SpecimenOrObservationBase> results = new ArrayList<>();
 278         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 279             results = dao.list(type, determinedAs, pageSize, pageNumber, orderHints, propertyPaths);
 280         }
 281         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 282     }
 283
 284     @Override
 285     public Pager<SpecimenOrObservationBase> list(Class<? extends SpecimenOrObservationBase> type, TaxonBase determinedAs,
 286             Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 287         long numberOfResults = dao.count(type, determinedAs);
 288         @SuppressWarnings("rawtypes")
 289         List<SpecimenOrObservationBase> results = new ArrayList<>();
 290         if(numberOfResults > 0) { // no point checking again  //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 291             results = dao.list(type, determinedAs, pageSize, pageNumber, orderHints, propertyPaths);
 292         }
 293         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 294     }
 295
 296     @Override
 297     public List<UuidAndTitleCache<DerivedUnit>> getDerivedUnitUuidAndTitleCache(Integer limit, String pattern) {
 298         return dao.getDerivedUnitUuidAndTitleCache(limit, pattern);
 299     }
 300
 301     @Override
 302     public List<UuidAndTitleCache<FieldUnit>> getFieldUnitUuidAndTitleCache() {
 303         return dao.getFieldUnitUuidAndTitleCache();
 304     }
 305
 306     @Override
 307     public DerivedUnitFacade getDerivedUnitFacade(DerivedUnit derivedUnit, List<String> derivedUnitFacadeInitStrategy) throws DerivedUnitFacadeNotSupportedException {
 308         derivedUnit = (DerivedUnit) dao.load(derivedUnit.getUuid(), null);
 309         DerivedUnitFacadeConfigurator config = DerivedUnitFacadeConfigurator.NewInstance();
 310         config.setThrowExceptionForNonSpecimenPreservationMethodRequest(false);
 311         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(derivedUnit, config);
 312         beanInitializer.initialize(derivedUnitFacade, derivedUnitFacadeInitStrategy);
 313         return derivedUnitFacade;
 314     }
 315
 316     @Override
 317     public <T extends SpecimenOrObservationBase> List<T> listByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 318             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 319
 320         return pageByAssociatedTaxon(type, includeRelationships, associatedTaxon, maxDepth, pageSize, pageNumber, orderHints, propertyPaths).getRecords();
 321     }
 322
 323     @Override
 324     public Collection<SpecimenNodeWrapper> listUuidAndTitleCacheByAssociatedTaxon(List<UUID> taxonNodeUuids,
 325             Integer limit, Integer start) {
 326         return dao.listUuidAndTitleCacheByAssociatedTaxon(taxonNodeUuids, limit, start);
 327     }
 328
 329     @Override
 330     public <T extends SpecimenOrObservationBase> Collection<T> listRootUnitsByAssociatedTaxon(Class<T> type, Taxon associatedTaxon, List<OrderHint> orderHints, List<String> propertyPaths) {
 331         return pageRootUnitsByAssociatedTaxon(type, null, associatedTaxon, null, null, null, null, propertyPaths).getRecords();
 332     }
 333
 334     @Override
 335     public <T extends SpecimenOrObservationBase> Pager<T> pageRootUnitsByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 336             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints,
 337             List<String> propertyPaths) {
 338
 339         if (!getSession().contains(associatedTaxon)) {
 340             associatedTaxon = (Taxon) taxonService.load(associatedTaxon.getUuid());
 341         }
 342
 343         // gather the IDs of all relevant root units
 344         Set<UUID> rootUnitUuids = new HashSet<>();
 345         List<SpecimenOrObservationBase> records = listByAssociatedTaxon(null, includeRelationships, associatedTaxon, maxDepth, null, null, orderHints, propertyPaths);
 346         for (SpecimenOrObservationBase<?> specimen : records) {
 347             for (SpecimenOrObservationBase<?> rootUnit : findRootUnits(specimen.getUuid(), null)) {
 348                 if(type == null || type.isAssignableFrom(rootUnit.getClass())) {
 349                     rootUnitUuids.add(rootUnit.getUuid());
 350                 }
 351             }
 352         }
 353         long totalCount = rootUnitUuids.size();
 354         //dao.list() does the paging of the field units. Passing the field units directly to the Pager would not work
 355         List<SpecimenOrObservationBase> rootUnits = dao.list(rootUnitUuids, pageSize, pageNumber, orderHints, propertyPaths);
 356         List<T> castedUnits = new ArrayList<>(rootUnits.size());
 357         for(SpecimenOrObservationBase<?> sob : rootUnits) {
 358             // this cast should be save since the uuids have been filtered by type above
 359             castedUnits.add((T)sob);
 360         }
 361         return new DefaultPagerImpl<>(pageNumber, totalCount, pageSize, castedUnits);
 362     }
 363
 364     @Override
 365     public FieldUnitDTO assembleFieldUnitDTO(FieldUnit fieldUnit) {
 366
 367         if (!getSession().contains(fieldUnit)) {
 368             fieldUnit = (FieldUnit) load(fieldUnit.getUuid());
 369         }
 370         // FIXME the filter for SpecimenOrObservationType.PreservedSpecimen has been preserved from the former implementation (see commit 07e3f63c7d  and older)
 371         // it is questionable if this filter makes sense for all use cases or if it is only a sensible default for the
 372         // compressed specimen table in the cdm-dataportal (see #6816, #6870)
 373         EnumSet<SpecimenOrObservationType> typeIncludeFilter = EnumSet.of(SpecimenOrObservationType.PreservedSpecimen);
 374         FieldUnitDTO fieldUnitDTO = FieldUnitDTO.fromEntity(fieldUnit, null, typeIncludeFilter);
 375         return fieldUnitDTO;
 376     }
 377
 378
 379     @Override
 380     @Transactional
 381     public DerivedUnitDTO assembleDerivedUnitDTO(DerivedUnit derivedUnit) {
 382
 383         if (!getSession().contains(derivedUnit)) {
 384             derivedUnit = (DerivedUnit) load(derivedUnit.getUuid());
 385         }
 386         DerivedUnitDTO derivedUnitDTO = DerivedUnitDTO.fromEntity(derivedUnit, null, null);
 387
 388         // individuals associations
 389         Collection<IndividualsAssociation> individualsAssociations = listIndividualsAssociations(derivedUnit, null, null, null, null);
 390         if(individualsAssociations != null) {
 391             for (IndividualsAssociation individualsAssociation : individualsAssociations) {
 392                 if (individualsAssociation.getInDescription() != null) {
 393                     if (individualsAssociation.getInDescription().isInstanceOf(TaxonDescription.class)) {
 394                         TaxonDescription taxonDescription = HibernateProxyHelper.deproxy(individualsAssociation.getInDescription(), TaxonDescription.class);
 395                         Taxon taxon = taxonDescription.getTaxon();
 396                         if (taxon != null) {
 397                             derivedUnitDTO.addAssociatedTaxon(taxon);
 398                         }
 399                     }
 400                 }
 401             }
 402         }
 403
 404         return derivedUnitDTO;
 405     }
 406
 407     @Override
 408
 409     public String getMostSignificantIdentifier(UUID derivedUnitUuid) {
 410         return dao.findMostSignificantIdentifier(derivedUnitUuid);
 411     }
 412
 413     /**
 414      * TODO there is a very similar function in {@link SpecimenOrObservationBaseDTO#assembleDerivative}.
 415      * If possible we should avoid using this function here by the method in <code>SpecimenOrObservationBaseDTO</code>.
 416      */
 417     private Set<DerivedUnitDTO> getDerivedUnitDTOsFor(SpecimenOrObservationBaseDTO specimenDto, DerivedUnit specimen,
 418             HashMap<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 419
 420         Set<DerivedUnitDTO> derivedUnits = new HashSet<>();
 421         for (DerivationEvent derivationEvent : specimen.getDerivationEvents()) {
 422             for (DerivedUnit derivative : derivationEvent.getDerivatives()) {
 423                 if (!alreadyCollectedSpecimen.containsKey(specimenDto.getUuid())){
 424                     DerivedUnitDTO dto = (DerivedUnitDTO) SpecimenOrObservationDTOFactory.fromEntity(derivative, 0);
 425                     alreadyCollectedSpecimen.put(dto.getUuid(), dto);
 426                     dto.addAllDerivatives(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 427                     derivedUnits.add(dto);
 428                 } else {
 429                     if(alreadyCollectedSpecimen.get(specimenDto.getUuid()).getDerivatives().isEmpty() && !derivative.getDerivationEvents().isEmpty()) {
 430                         // we need to add the missing derivatives!
 431                         SpecimenOrObservationBaseDTO dto = alreadyCollectedSpecimen.get(specimenDto.getUuid());
 432                         alreadyCollectedSpecimen.get(specimenDto.getUuid()).addAllDerivatives(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 433                     }
 434                 }
 435             }
 436         }
 437         return derivedUnits;
 438     }
 439
 440 //    private Set<DerivateDTO> getDerivedUnitDTOsFor(DerivateDTO specimenDto, DerivedUnit specimen, HashMap<UUID, DerivateDTO> alreadyCollectedSpecimen) {
 441 //        Set<DerivateDTO> derivedUnits = new HashSet<>();
 442 ////        load
 443 //        for (DerivationEvent derivationEvent : specimen.getDerivationEvents()) {
 444 //            for (DerivedUnit derivative : derivationEvent.getDerivatives()) {
 445 //                if (!alreadyCollectedSpecimen.containsKey(specimenDto.getUuid())){
 446 //                    PreservedSpecimenDTO dto;
 447 //                    if (derivative instanceof DnaSample){
 448 //                        dto = DNASampleDTO.newInstance(derivative);
 449 //                    }else{
 450 //                        dto = PreservedSpecimenDTO.newInstance(derivative);
 451 //                    }
 452 //                    alreadyCollectedSpecimen.put(dto.getUuid(), dto);
 453 //                    dto.addAllDerivates(getDerivedUnitDTOsFor(dto, derivative, alreadyCollectedSpecimen));
 454 //                    derivedUnits.add(dto);
 455 //                }
 456 //            }
 457 //        }
 458 //        return derivedUnits;
 459 //    }
 460
 461
 462     @SuppressWarnings("unchecked")
 463     @Override
 464     public <T extends SpecimenOrObservationBase> Pager<T> pageByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includedRelationships,
 465             Taxon associatedTaxon, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 466
 467         Set<Taxon> taxa = new HashSet<>();
 468         Set<Integer> occurrenceIds = new HashSet<>();
 469         List<T> occurrences = new ArrayList<>();
 470         boolean includeUnpublished = INCLUDE_UNPUBLISHED;
 471
 472         // Integer limit = PagerUtils.limitFor(pageSize);
 473         // Integer start = PagerUtils.startFor(pageSize, pageNumber);
 474
 475         if (!getSession().contains(associatedTaxon)) {
 476             associatedTaxon = (Taxon) taxonService.load(associatedTaxon.getUuid());
 477         }
 478
 479         if (includedRelationships != null) {
 480             taxa = taxonService.listRelatedTaxa(associatedTaxon, includedRelationships, maxDepth, includeUnpublished, null, null, propertyPaths);
 481         }
 482
 483         taxa.add(associatedTaxon);
 484
 485         for (Taxon taxon : taxa) {
 486             List<T> perTaxonOccurrences = dao.listByAssociatedTaxon(type, taxon, includeUnpublished,
 487                     null, null, orderHints, propertyPaths);
 488             for (SpecimenOrObservationBase<?> o : perTaxonOccurrences) {
 489                 occurrenceIds.add(o.getId());
 490             }
 491         }
 492         occurrences = (List<T>) dao.loadList(occurrenceIds, null, propertyPaths);
 493
 494         return new DefaultPagerImpl<>(pageNumber, Long.valueOf(occurrences.size()), pageSize, occurrences);
 495
 496     }
 497
 498     @Override
 499     public <T extends SpecimenOrObservationBase> Pager<T> pageByAssociatedTaxon(Class<T> type, Set<TaxonRelationshipEdge> includeRelationships,
 500             String taxonUUID, Integer maxDepth, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 501
 502         UUID uuid = UUID.fromString(taxonUUID);
 503         Taxon taxon = (Taxon) taxonService.load(uuid);
 504         return pageByAssociatedTaxon(type, includeRelationships, taxon, maxDepth, pageSize, pageNumber, orderHints, propertyPaths);
 505     }
 506
 507     @Override
 508     @Transactional
 509     public List<SpecimenOrObservationBaseDTO> listRootUnitDTOsByAssociatedTaxon(Set<TaxonRelationshipEdge> includedRelationships,
 510             UUID associatedTaxonUuid, boolean includeUnpublished, List<String> propertyPaths) {
 511
 512         Set<Taxon> taxa = new HashSet<>();
 513         Set<SpecimenOrObservationBaseDTO> rootUnitDTOs = new HashSet<>();
 514
 515         Taxon associatedTaxon = (Taxon) taxonService.load(associatedTaxonUuid);
 516         if (includedRelationships != null) {
 517             taxa = taxonService.listRelatedTaxa(associatedTaxon, includedRelationships, null, includeUnpublished, null, null, null);
 518         }
 519         taxa.add(associatedTaxon);
 520
 521         HashMap<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedUnits = new HashMap<>();
 522         for (Taxon taxon : taxa) {
 523             // TODO there might be a good potential to speed up the whole processing by collecting all entities first
 524             // and to create the DTOs in a second step
 525             Set<SpecimenOrObservationBase> perTaxonOccurrences = dao.listByAssociatedTaxon(
 526                     null, taxon, includeUnpublished, null, null, null, propertyPaths)
 527                     .stream()
 528                     .map(u -> HibernateProxyHelper.deproxy(u, SpecimenOrObservationBase.class))
 529                     .collect(Collectors.toSet());
 530             for (SpecimenOrObservationBase<?> unit : perTaxonOccurrences) {
 531                 unit = HibernateProxyHelper.deproxy(unit);
 532                 if (unit instanceof DerivedUnit){
 533                     DerivedUnitDTO derivativeDTO;
 534                     if (!alreadyCollectedUnits.containsKey(unit.getUuid())){
 535                         DerivedUnit derivedUnit = (DerivedUnit)unit;
 536                         boolean isAssociated = false;
 537                         for (DeterminationEvent determination:derivedUnit.getDeterminations()) {
 538                                 if (determination.getTaxonName() != null && determination.getTaxonName().equals(taxon.getName()) || taxon.equals(determination.getTaxon()) ){
 539                                         isAssociated = true;
 540                                         break;
 541                                 }
 542                         }
 543                         for (TaxonDescription desc: taxon.getDescriptions()) {
 544                             for (DescriptionElementBase descElement: desc.getElements()) {
 545                                 if (descElement instanceof IndividualsAssociation) {
 546                                     if (((IndividualsAssociation)descElement).getAssociatedSpecimenOrObservation().getUuid().equals(derivedUnit.getUuid())) {
 547                                         isAssociated = true;
 548                                         break;
 549                                     }
 550                                 }
 551                             }
 552                         }
 553
 554                         for (TypeDesignationBase<?> desc: taxon.getName().getHomotypicalGroup().getTypeDesignations()) {
 555                             if (desc instanceof SpecimenTypeDesignation) {
 556                                 if (((SpecimenTypeDesignation)desc).getTypeSpecimen().getUuid().equals(derivedUnit.getUuid())) {
 557                                     isAssociated = true;
 558                                     break;
 559                                 }
 560                             }
 561                         }
 562
 563                         for (TaxonName name: taxon.getSynonymNames()) {
 564                             for (TypeDesignationBase<?> desc :name.getHomotypicalGroup().getTypeDesignations()) {
 565
 566                                 if (desc instanceof SpecimenTypeDesignation) {
 567                                     if (((SpecimenTypeDesignation)desc).getTypeSpecimen().getUuid().equals(derivedUnit.getUuid())) {
 568                                         isAssociated = true;
 569                                         break;
 570                                     }
 571                                 }
 572                             }
 573                         }
 574
 575                         if (!isAssociated) {
 576                                 continue;
 577                         }
 578                         derivativeDTO = (DerivedUnitDTO) SpecimenOrObservationDTOFactory.fromEntity(derivedUnit, null);
 579                         alreadyCollectedUnits.put(derivativeDTO.getUuid(), derivativeDTO);
 580                         //derivativeDTO.addAllDerivatives(getDerivedUnitDTOsFor(derivativeDTO, derivedUnit, alreadyCollectedUnits));
 581                     }
 582                     derivativeDTO = (DerivedUnitDTO) alreadyCollectedUnits.get(unit.getUuid());
 583                     rootUnitDTOs.addAll(findRootUnitDTOs(derivativeDTO, alreadyCollectedUnits));
 584                 } else {
 585                     // only other option is FieldUnit
 586                     rootUnitDTOs.add(FieldUnitDTO.fromEntity((FieldUnit)unit, 0, null));
 587                 }
 588             }
 589         }
 590
 591         List<SpecimenOrObservationBaseDTO> orderdDTOs = new ArrayList<>(rootUnitDTOs);
 592         // TODO order dtos by date can only be done by string comparison
 593         // the FieldUnitDTO.date needs to be a Partial object for sensible ordering
 594         Collections.sort(orderdDTOs, new Comparator<SpecimenOrObservationBaseDTO>() {
 595
 596             @Override
 597             public int compare(SpecimenOrObservationBaseDTO o1, SpecimenOrObservationBaseDTO o2) {
 598                 if(o1 instanceof FieldUnitDTO && o2 instanceof FieldUnitDTO) {
 599                     FieldUnitDTO fu1 = (FieldUnitDTO)o1;
 600                     FieldUnitDTO fu2 = (FieldUnitDTO)o2;
 601                     //TODO if we want null values and values with missing year as smallest we should use
 602                     //PartialComparator.INSTANCE_NULL_SMALLEST() here
 603                     return PartialComparator.INSTANCE().compare(fu1.getDate(), fu2.getDate());
 604                 }
 605                 if(o1 instanceof DerivedUnitDTO && o2 instanceof DerivedUnitDTO) {
 606                     SpecimenOrObservationBaseDTO du1 = o1;
 607                     SpecimenOrObservationBaseDTO du2 = o2;
 608                     return StringUtils.compare(du1.getLabel(), du2.getLabel());
 609                  }
 610                 if(o1 instanceof FieldUnitDTO && o2 instanceof DerivedUnitDTO) {
 611                     return -1;
 612                 } else {
 613                     return 1;
 614                 }
 615             }
 616
 617         });
 618
 619         return orderdDTOs;
 620     }
 621
 622     @Override
 623     @Transactional
 624     public  SpecimenOrObservationBaseDTO findByGeneticAccessionNumber(String accessionNumberString, List<OrderHint> orderHints)  {
 625
 626         DnaSample dnaSample = dao.findByGeneticAccessionNumber(accessionNumberString, null);
 627         DerivedUnitDTO dnaSampleDTO;
 628         if (dnaSample != null){
 629             dnaSampleDTO = new DNASampleDTO(dnaSample);
 630             Collection<SpecimenOrObservationBaseDTO> fieldUnitDTOs = this.findRootUnitDTOs(dnaSampleDTO, new HashMap<>());
 631             // FIXME change return type to Collection<FieldUnitDTO>
 632             if(fieldUnitDTOs.isEmpty()) {
 633                 return null;
 634             } else {
 635                return fieldUnitDTOs.iterator().next();
 636             }
 637         }
 638         return null;
 639     }
 640
 641     @Override
 642     public Pager<SearchResult<SpecimenOrObservationBase>> findByFullText(
 643             Class<? extends SpecimenOrObservationBase> clazz, String queryString, RectangleDTO boundingBox, List<Language> languages,
 644             boolean highlightFragments, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints,
 645             List<String> propertyPaths) throws IOException, LuceneParseException {
 646
 647         LuceneSearch luceneSearch = prepareByFullTextSearch(clazz, queryString, boundingBox, languages, highlightFragments);
 648
 649         // --- execute search
 650         TopGroups<BytesRef> topDocsResultSet;
 651         try {
 652             topDocsResultSet = luceneSearch.executeSearch(pageSize, pageNumber);
 653         } catch (ParseException e) {
 654             LuceneParseException parseException = new LuceneParseException(e.getMessage());
 655             parseException.setStackTrace(e.getStackTrace());
 656             throw parseException;
 657         }
 658
 659         Map<CdmBaseType, String> idFieldMap = new HashMap<>();
 660         idFieldMap.put(CdmBaseType.SPECIMEN_OR_OBSERVATIONBASE, "id");
 661
 662         // --- initialize taxa, highlight matches ....
 663         ISearchResultBuilder searchResultBuilder = new SearchResultBuilder(luceneSearch, luceneSearch.getQuery());
 664         @SuppressWarnings("rawtypes")
 665         List<SearchResult<SpecimenOrObservationBase>> searchResults = searchResultBuilder.createResultSet(
 666                 topDocsResultSet, luceneSearch.getHighlightFields(), dao, idFieldMap, propertyPaths);
 667
 668         int totalHits = topDocsResultSet != null ? topDocsResultSet.totalGroupCount : 0;
 669
 670         return new DefaultPagerImpl<>(pageNumber, Long.valueOf(totalHits), pageSize, searchResults);
 671     }
 672
 673     private LuceneSearch prepareByFullTextSearch(Class<? extends SpecimenOrObservationBase> clazz, String queryString, RectangleDTO bbox,
 674             List<Language> languages, boolean highlightFragments) {
 675
 676         Builder finalQueryBuilder = new Builder();
 677         Builder textQueryBuilder = new Builder();
 678
 679         LuceneSearch luceneSearch = new LuceneSearch(luceneIndexToolProvider, FieldUnit.class);
 680         QueryFactory queryFactory = luceneIndexToolProvider.newQueryFactoryFor(FieldUnit.class);
 681
 682         // --- criteria
 683         luceneSearch.setCdmTypRestriction(clazz);
 684         if (queryString != null) {
 685             textQueryBuilder.add(queryFactory.newTermQuery("titleCache", queryString), Occur.SHOULD);
 686             finalQueryBuilder.add(textQueryBuilder.build(), Occur.MUST);
 687         }
 688
 689         // --- spacial query
 690         if (bbox != null) {
 691             finalQueryBuilder.add(QueryFactory.buildSpatialQueryByRange(bbox, "gatheringEvent.exactLocation.point"), Occur.MUST);
 692         }
 693
 694         luceneSearch.setQuery(finalQueryBuilder.build());
 695
 696         // --- sorting
 697         SortField[] sortFields = new SortField[] { SortField.FIELD_SCORE, new SortField("titleCache__sort", SortField.Type.STRING, false) };
 698         luceneSearch.setSortFields(sortFields);
 699
 700         if (highlightFragments) {
 701             luceneSearch.setHighlightFields(queryFactory.getTextFieldNamesAsArray());
 702         }
 703         return luceneSearch;
 704     }
 705
 706
 707     @Override
 708     @Transactional(readOnly=true)
 709     public Collection<FieldUnit> findFieldUnits(UUID derivedUnitUuid, List<String> propertyPaths) {
 710         //It will search recursively over all {@link DerivationEvent}s and get the "originals" ({@link SpecimenOrObservationBase})
 711         //from which this DerivedUnit was derived until all FieldUnits are found.
 712
 713         // FIXME: use HQL queries to avoid entity instantiation and to increase performance
 714
 715         SpecimenOrObservationBase<?> specimen = load(derivedUnitUuid);
 716         Collection<FieldUnit> fieldUnits = new ArrayList<>();
 717         if (specimen == null){
 718             return null;
 719         }
 720         if (specimen.isInstanceOf(FieldUnit.class)) {
 721             fieldUnits.add(HibernateProxyHelper.deproxy(specimen, FieldUnit.class));
 722         }
 723         else if(specimen.isInstanceOf(DerivedUnit.class)){
 724             fieldUnits.addAll(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class).collectRootUnits(FieldUnit.class));
 725         }
 726
 727         fieldUnits = beanInitializer.initializeAll(fieldUnits, propertyPaths);
 728         return fieldUnits;
 729     }
 730
 731     @Override
 732     @Transactional(readOnly=true)
 733     public Collection<SpecimenOrObservationBase> findRootUnits(UUID derivedUnitUuid, List<String> propertyPaths) {
 734
 735         // FIXME: use HQL queries to avoid entity instantiation and to increase performance
 736
 737         SpecimenOrObservationBase<?> specimen = load(derivedUnitUuid);
 738         Collection<SpecimenOrObservationBase> rootUnits = new ArrayList<>();
 739         if (specimen == null){
 740             return null;
 741         }
 742         if (specimen.isInstanceOf(FieldUnit.class)) {
 743             rootUnits.add(HibernateProxyHelper.deproxy(specimen, FieldUnit.class));
 744         }
 745         else if(specimen.isInstanceOf(DerivedUnit.class)){
 746             rootUnits.addAll(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class).collectRootUnits(SpecimenOrObservationBase.class));
 747         }
 748
 749         rootUnits = beanInitializer.initializeAll(rootUnits, propertyPaths);
 750         return rootUnits;
 751     }
 752
 753     @Override
 754     public Collection<SpecimenOrObservationBaseDTO> findRootUnitDTOs(UUID unitUUID) {
 755
 756         SpecimenOrObservationBase<?> entity = load(unitUUID);
 757         SpecimenOrObservationBaseDTO derivedUnitDTO = SpecimenOrObservationDTOFactory.fromEntity(entity);
 758         Collection<SpecimenOrObservationBaseDTO> rootUnitDTOs = new ArrayList<>();
 759         if(derivedUnitDTO != null) {
 760             if(derivedUnitDTO instanceof FieldUnitDTO) {
 761                 rootUnitDTOs.add(derivedUnitDTO);
 762             } else {
 763                 Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen = new HashMap<>();
 764                 rootUnitDTOs  = findRootUnitDTOs((DerivedUnitDTO)derivedUnitDTO, alreadyCollectedSpecimen);
 765             }
 766         }
 767
 768         return rootUnitDTOs;
 769
 770     }
 771
 772     /**
 773      * Recursively searches all {@link DerivationEvent}s to find all "originals" ({@link SpecimenOrObservationBase})
 774      * from which this DerivedUnit was derived until all FieldUnits are found.
 775      * <p>
 776      * <b>NOTE:</b> The recursive search still is a bit incomplete and may miss originals in the rare case where a
 777      * derivative has more than one original. (see https://dev.e-taxonomy.eu/redmine/issues/9253)
 778      *
 779      * @param derivedUnitDTO
 780      *  The DerivedUnitDTO to start the search from.
 781      * @param alreadyCollectedSpecimen
 782      *  A map to hold all originals that have been sees during the recursive walk.
 783      * @return
 784      *  The collection of all Field Units that are accessible from the derivative from where the search was started.
 785      */
 786     public Collection<SpecimenOrObservationBaseDTO> findRootUnitDTOs(DerivedUnitDTO derivedUnitDTO,
 787             Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 788
 789         HashMap<UUID, SpecimenOrObservationBaseDTO> rootUnitDTOs = new HashMap<>();
 790         _findRootUnitDTOs(derivedUnitDTO, rootUnitDTOs, alreadyCollectedSpecimen);
 791         return rootUnitDTOs.values();
 792
 793     }
 794
 795     /**
 796      * Method for recursive calls, must only be used by {@link #findRootUnitDTOs(DerivedUnitDTO, HashMap)}
 797      * <p>
 798      * It will search recursively over all {@link DerivationEvent}s and get the "originals" ({@link SpecimenOrObservationBase})
 799      * from which this DerivedUnit was derived until all FieldUnits are found.
 800      */
 801     private void _findRootUnitDTOs(DerivedUnitDTO derivedUnitDTO, Map<UUID, SpecimenOrObservationBaseDTO> rootUnitDTOs,
 802                 Map<UUID, SpecimenOrObservationBaseDTO> alreadyCollectedSpecimen) {
 803
 804         List<String> propertyPaths = new ArrayList<>();
 805
 806         // add the supplied DTO the the alreadyCollectedSpecimen if not yet there
 807         if(!alreadyCollectedSpecimen.containsKey(derivedUnitDTO.getUuid())) {
 808             alreadyCollectedSpecimen.put(derivedUnitDTO.getUuid(), derivedUnitDTO);
 809         }
 810
 811         List<SpecimenOrObservationBase> originals = dao.findOriginalsForDerivedUnit(derivedUnitDTO.getUuid(), propertyPaths);
 812         if (originals.size() > 0){
 813             if (originals.size() > 1){
 814                 logger.warn("The derived unit with uuid " + derivedUnitDTO.getUuid() + "has more than one orginal");
 815             }
 816             // FIXME allow handling multiple originals
 817             SpecimenOrObservationBase<?> original = originals.get(0);
 818             original = HibernateProxyHelper.deproxy(original);
 819
 820             if (alreadyCollectedSpecimen.containsKey(original.getUuid())){
 821                 alreadyCollectedSpecimen.get(original.getUuid()).addDerivative(derivedUnitDTO);
 822     //            if ( alreadyCollectedSpecimen.get(specimen.getUuid()) instanceof FieldUnitDTO){
 823     //                ((FieldUnitDTO)alreadyCollectedSpecimen.get(specimen.getUuid())).getTaxonRelatedDerivedUnits().add(derivedUnitDTO.getUuid());
 824     //            }
 825             }else{
 826                 if(!rootUnitDTOs.containsKey(original.getUuid())){
 827                     // the direct derivatives of the field unit are added in the factory method, so it is guaranteed that
 828                     // the derivedUnitDTO is already contained.
 829                     // ----
 830                     // Don't assemble derivatives for originals since we have them collected already
 831                     // when ascending to the originals, we only want to collect those derivatives which are on the path up to the root
 832                     final Integer maxDepth = 0;
 833                     SpecimenOrObservationBaseDTO originalDTO = SpecimenOrObservationDTOFactory.fromEntity(original, maxDepth);
 834                     originalDTO.addDerivative(derivedUnitDTO);
 835                     alreadyCollectedSpecimen.put(originalDTO.getUuid(), originalDTO);
 836                     if (original instanceof FieldUnit){
 837                         rootUnitDTOs.put(originalDTO.getUuid(), originalDTO);
 838                     }else{
 839                         _findRootUnitDTOs((DerivedUnitDTO) originalDTO, rootUnitDTOs, alreadyCollectedSpecimen);
 840                     }
 841                 } else {
 842                     SpecimenOrObservationBaseDTO previouslyFoundRootUnit = rootUnitDTOs.get(original.getUuid());
 843                     if(!previouslyFoundRootUnit.getDerivatives().stream().anyMatch(uDTO -> uDTO.getUuid().equals(derivedUnitDTO.getUuid()))) {
 844                         previouslyFoundRootUnit.addDerivative(derivedUnitDTO);
 845                     }
 846                 }
 847             }
 848         } else {
 849             rootUnitDTOs.put(derivedUnitDTO.getUuid(), derivedUnitDTO);
 850         }
 851
 852     }
 853
 854     @Override
 855     @Transactional(readOnly=true)
 856     public FieldUnitDTO loadFieldUnitDTO(UUID derivedUnitUuid) {
 857
 858         FieldUnitDTO fieldUnitDTO = null;
 859         DerivedUnitDTO derivedUnitDTO = null;
 860
 861         Map<UUID, SpecimenOrObservationBaseDTO> cycleDetectionMap = new HashMap<>();
 862         SpecimenOrObservationBase<?> derivative = dao.load(derivedUnitUuid);
 863         if(derivative != null){
 864             if (derivative instanceof FieldUnit){
 865                 fieldUnitDTO = FieldUnitDTO.fromEntity((FieldUnit)derivative);
 866                 return fieldUnitDTO;
 867             } else {
 868                 // must be a DerivedUnit otherwise
 869                 derivedUnitDTO = DerivedUnitDTO.fromEntity((DerivedUnit)derivative);
 870                 while(true){
 871
 872                     Set<SpecimenOrObservationBaseDTO> originals = originalDTOs(derivedUnitDTO.getUuid());
 873
 874                     if(originals.isEmpty()){
 875                         break;
 876                     }
 877                     if (originals.size() > 1){
 878                         logger.debug("The derived unit with uuid " + derivedUnitUuid + "has more than one orginal, ignoring all but the first one.");
 879                     }
 880
 881                     SpecimenOrObservationBaseDTO originalDTO = originals.iterator().next();
 882
 883                     // cycle detection and handling
 884                     if(cycleDetectionMap.containsKey(originalDTO.getUuid())){
 885                         // cycle detected!!!
 886                         try {
 887                             throw new Exception();
 888                         } catch(Exception e){
 889                             logger.error("Cycle in derivate graph detected at DerivedUnit with uuid=" + originalDTO.getUuid() , e);
 890                         }
 891                         // to solve the situation for the output we remove the derivate from the more distant graph node
 892                         // by removing it from the derivatives of its original
 893                         // but let the derivate to be added to the original which is closer to the FieldUnit (below at originalDTO.addDerivate(derivedUnitDTO);)
 894                         for(SpecimenOrObservationBaseDTO seenOriginal: cycleDetectionMap.values()){
 895                             for(SpecimenOrObservationBaseDTO derivateDTO : seenOriginal.getDerivatives()){
 896                                 if(derivateDTO.equals(originalDTO)){
 897                                     seenOriginal.getDerivatives().remove(originalDTO);
 898                                 }
 899                             }
 900                         }
 901                     } else {
 902                         cycleDetectionMap.put(originalDTO.getUuid(), originalDTO);
 903                     }
 904
 905                     if (originalDTO instanceof FieldUnitDTO){
 906                         fieldUnitDTO = (FieldUnitDTO)originalDTO;
 907                         break;
 908                     }else{
 909                         // So this must be a DerivedUnitDTO
 910                         if (derivedUnitDTO == null){
 911                             derivedUnitDTO = (DerivedUnitDTO)originalDTO;
 912                         } else {
 913                             derivedUnitDTO = (DerivedUnitDTO)originalDTO;
 914                         }
 915                     }
 916                 }
 917             }
 918         }
 919         return fieldUnitDTO;
 920
 921     }
 922
 923     /**
 924      * @param originalDTO
 925      * @return
 926      */
 927     private Set<SpecimenOrObservationBaseDTO> originalDTOs(UUID derivativeUuid) {
 928
 929         Set<SpecimenOrObservationBaseDTO> dtos = new HashSet<>();
 930         List<SpecimenOrObservationBase> specimens = dao.findOriginalsForDerivedUnit(derivativeUuid, null);
 931         for(SpecimenOrObservationBase sob : specimens){
 932             if(sob instanceof FieldUnit) {
 933                 dtos.add(FieldUnitDTO.fromEntity((FieldUnit)sob));
 934             } else {
 935                 dtos.add(DerivedUnitDTO.fromEntity((DerivedUnit)sob));
 936             }
 937         }
 938         return dtos;
 939     }
 940
 941
 942     @Override
 943     @Transactional(readOnly = false)
 944     public UpdateResult moveSequence(DnaSample from, DnaSample to, Sequence sequence) {
 945         return moveSequence(from.getUuid(), to.getUuid(), sequence.getUuid());
 946     }
 947
 948     @Override
 949     @Transactional(readOnly = false)
 950     public UpdateResult moveSequence(UUID fromUuid, UUID toUuid, UUID sequenceUuid) {
 951         // reload specimens to avoid session conflicts
 952         DnaSample from = (DnaSample) load(fromUuid);
 953         DnaSample to = (DnaSample) load(toUuid);
 954         Sequence sequence = sequenceService.load(sequenceUuid);
 955
 956         if (from == null || to == null || sequence == null) {
 957             throw new TransientObjectException("One of the CDM entities has not been saved to the data base yet. Moving only works for persisted/saved CDM entities.\n" +
 958                     "Operation was move "+sequence+ " from "+from+" to "+to);
 959         }
 960         UpdateResult result = new UpdateResult();
 961         from.removeSequence(sequence);
 962         saveOrUpdate(from);
 963         to.addSequence(sequence);
 964         saveOrUpdate(to);
 965         result.setStatus(Status.OK);
 966         result.addUpdatedObject(from);
 967         result.addUpdatedObject(to);
 968         return result;
 969     }
 970
 971     @Override
 972     @Transactional(readOnly = false)
 973     public boolean moveDerivate(SpecimenOrObservationBase<?> from, SpecimenOrObservationBase<?> to, DerivedUnit derivate) {
 974         return moveDerivate(from!=null?from.getUuid():null, to.getUuid(), derivate.getUuid()).isOk();
 975     }
 976
 977     @Override
 978     @Transactional(readOnly = false)
 979     public UpdateResult moveDerivate(UUID specimenFromUuid, UUID specimenToUuid, UUID derivateUuid) {
 980         // reload specimens to avoid session conflicts
 981         SpecimenOrObservationBase<?> from = null;
 982         if(specimenFromUuid!=null){
 983             from = load(specimenFromUuid);
 984         }
 985         SpecimenOrObservationBase<?> to = load(specimenToUuid);
 986         DerivedUnit derivate = (DerivedUnit) load(derivateUuid);
 987
 988         if ((specimenFromUuid!=null && from == null) || to == null || derivate == null) {
 989             throw new TransientObjectException("One of the CDM entities has not been saved to the data base yet. Moving only works for persisted/saved CDM entities.\n" +
 990                     "Operation was move "+derivate+ " from "+from+" to "+to);
 991         }
 992         UpdateResult result = new UpdateResult();
 993         SpecimenOrObservationType derivateType = derivate.getRecordBasis();
 994         SpecimenOrObservationType toType = to.getRecordBasis();
 995         // check if type is a sub derivate type
 996         if(toType==SpecimenOrObservationType.FieldUnit //moving to FieldUnit always works
 997                 || derivateType==SpecimenOrObservationType.Media //moving media always works
 998                 || (derivateType.isKindOf(toType) && toType!=derivateType)){ //moving only to parent derivate type
 999             if(from!=null){
1000                 // remove derivation event from parent specimen of dragged object
1001                 DerivationEvent eventToRemove = null;
1002                 for (DerivationEvent event : from.getDerivationEvents()) {
1003                     if (event.getDerivatives().contains(derivate)) {
1004                         eventToRemove = event;
1005                         break;
1006                     }
1007                 }
1008                 from.removeDerivationEvent(eventToRemove);
1009                 if(eventToRemove!=null){
1010                     // add new derivation event to target and copy the event parameters of the old one
1011                     DerivationEvent derivedFromNewOriginalEvent = DerivationEvent.NewSimpleInstance(to, derivate, null);
1012                     derivedFromNewOriginalEvent.setActor(eventToRemove.getActor());
1013                     derivedFromNewOriginalEvent.setDescription(eventToRemove.getDescription());
1014                     derivedFromNewOriginalEvent.setInstitution(eventToRemove.getInstitution());
1015                     derivedFromNewOriginalEvent.setTimeperiod(eventToRemove.getTimeperiod());
1016                     derivedFromNewOriginalEvent.setType(eventToRemove.getType());
1017                     to.addDerivationEvent(derivedFromNewOriginalEvent);
1018                     derivate.setDerivedFrom(derivedFromNewOriginalEvent);
1019                 }
1020             }
1021             else{
1022                 //derivative had no parent before so we use empty derivation event
1023                 DerivationEvent derivedFromNewOriginalEvent = DerivationEvent.NewSimpleInstance(to, derivate, null);
1024                 to.addDerivationEvent(derivedFromNewOriginalEvent);
1025                 derivate.setDerivedFrom(derivedFromNewOriginalEvent);
1026             }
1027
1028             if(from!=null){
1029                 saveOrUpdate(from);
1030             }
1031             saveOrUpdate(to);
1032             result.setStatus(Status.OK);
1033             result.addUpdatedObject(from);
1034             result.addUpdatedObject(to);
1035         } else {
1036             result.setStatus(Status.ERROR);
1037         }
1038         return result;
1039     }
1040
1041     @Override
1042     public DeleteResult isDeletable(UUID specimenUuid, DeleteConfiguratorBase config) {
1043         DeleteResult deleteResult = new DeleteResult();
1044         SpecimenOrObservationBase<?> specimen = this.load(specimenUuid);
1045         SpecimenDeleteConfigurator specimenDeleteConfigurator = (SpecimenDeleteConfigurator) config;
1046
1047         // check elements found by super method
1048         Set<CdmBase> relatedObjects = super.isDeletable(specimenUuid, config).getRelatedObjects();
1049         for (CdmBase cdmBase : relatedObjects) {
1050             // check for type designation
1051             if (cdmBase.isInstanceOf(SpecimenTypeDesignation.class) && !specimenDeleteConfigurator.isDeleteFromTypeDesignation()) {
1052                 deleteResult.setAbort();
1053                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is a type specimen."));
1054                 deleteResult.addRelatedObject(cdmBase);
1055                 break;
1056             }
1057             // check for IndividualsAssociations
1058             else if (cdmBase.isInstanceOf(IndividualsAssociation.class) && !specimenDeleteConfigurator.isDeleteFromIndividualsAssociation()) {
1059                 deleteResult.setAbort();
1060                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is still associated via IndividualsAssociations"));
1061                 deleteResult.addRelatedObject(cdmBase);
1062                 break;
1063             }
1064             // check for taxon description
1065             else if(cdmBase.isInstanceOf(TaxonDescription.class)
1066                     && HibernateProxyHelper.deproxy(cdmBase, TaxonDescription.class).getDescribedSpecimenOrObservation().equals(specimen)
1067                     && !specimenDeleteConfigurator.isDeleteFromDescription()){
1068                 deleteResult.setAbort();
1069                 deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation is still used as \"Described Specimen\" in a taxon description."));
1070                 deleteResult.addRelatedObject(cdmBase);
1071                 break;
1072             }
1073             // check for children and parents (derivation events)
1074             else if (cdmBase.isInstanceOf(DerivationEvent.class)) {
1075                 DerivationEvent derivationEvent = HibernateProxyHelper.deproxy(cdmBase, DerivationEvent.class);
1076                 // check if derivation event is empty
1077                 if (!derivationEvent.getDerivatives().isEmpty() && derivationEvent.getOriginals().contains(specimen)) {
1078                     // if derivationEvent is the childEvent and contains derivations
1079 //                    if (derivationEvent.getDerivatives().contains(specimen)) {
1080 //                        //if it is the parent event the specimen is still deletable
1081 //                        continue;
1082 //                    }
1083                     if(!specimenDeleteConfigurator.isDeleteChildren()){
1084                         //if children should not be deleted then it is undeletable
1085                         deleteResult.setAbort();
1086                         deleteResult.addException(new ReferencedObjectUndeletableException("Specimen or obeservation still has child derivatives."));
1087                         deleteResult.addRelatedObject(cdmBase);
1088                         break;
1089                     }
1090                     else{
1091                         // check all children if they can be deleted
1092                         Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1093                         DeleteResult childResult = new DeleteResult();
1094                         for (DerivedUnit derivedUnit : derivatives) {
1095                             childResult.includeResult(isDeletable(derivedUnit.getUuid(), specimenDeleteConfigurator));
1096                         }
1097                         if (!childResult.isOk()) {
1098                             deleteResult.setAbort();
1099                             deleteResult.includeResult(childResult);
1100                             deleteResult.addRelatedObject(cdmBase);
1101                             break;
1102                         }
1103                     }
1104                 }
1105             }
1106             // check for amplification
1107             else if (cdmBase.isInstanceOf(AmplificationResult.class)
1108                     && !specimenDeleteConfigurator.isDeleteMolecularData()
1109                     && !specimenDeleteConfigurator.isDeleteChildren()) {
1110                 deleteResult.setAbort();
1111                 deleteResult.addException(new ReferencedObjectUndeletableException("DnaSample is used in amplification results."));
1112                 deleteResult.addRelatedObject(cdmBase);
1113                 break;
1114             }
1115             // check for sequence
1116             else if (cdmBase.isInstanceOf(Sequence.class)
1117                     && !specimenDeleteConfigurator.isDeleteMolecularData()
1118                     && !specimenDeleteConfigurator.isDeleteChildren()) {
1119                 deleteResult.setAbort();
1120                 deleteResult.addException(new ReferencedObjectUndeletableException("DnaSample is used in sequences."));
1121                 deleteResult.addRelatedObject(cdmBase);
1122                 break;
1123             }
1124         }
1125         if (deleteResult.isOk()) {
1126             //add all related object if deletion is OK so they can be handled by the delete() method
1127             deleteResult.addRelatedObjects(relatedObjects);
1128         }
1129         return deleteResult;
1130     }
1131
1132     @Transactional(readOnly = false)
1133     @Override
1134     public DeleteResult delete(UUID specimenUuid, SpecimenDeleteConfigurator config) {
1135         return delete(load(specimenUuid), config);
1136     }
1137
1138     @Transactional(readOnly = false)
1139     @Override
1140     public DeleteResult delete(SpecimenOrObservationBase<?> specimen, SpecimenDeleteConfigurator config) {
1141         specimen = HibernateProxyHelper.deproxy(specimen, SpecimenOrObservationBase.class);
1142
1143         DeleteResult deleteResult = isDeletable(specimen.getUuid(), config);
1144         if (!deleteResult.isOk()) {
1145             return deleteResult;
1146         }
1147
1148         if (config.isDeleteChildren()) {
1149             Set<DerivationEvent> derivationEvents = specimen.getDerivationEvents();
1150             //clone to avoid concurrent modification
1151             //can happen if the child is deleted and deleted its own derivedFrom event
1152             Set<DerivationEvent> derivationEventsClone = new HashSet<>(derivationEvents);
1153             for (DerivationEvent derivationEvent : derivationEventsClone) {
1154                 Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1155                 Iterator<DerivedUnit> it = derivatives.iterator();
1156                 Set<DerivedUnit> derivativesToDelete = new HashSet<>();
1157                 while (it.hasNext()) {
1158                     DerivedUnit unit = it.next();
1159                     derivativesToDelete.add(unit);
1160                 }
1161                 for (DerivedUnit unit: derivativesToDelete){
1162                     DeleteResult derivedDeleteResult = delete(unit, config);
1163                     deleteResult.includeResult(derivedDeleteResult);
1164                 }
1165             }
1166         }
1167
1168         // check related objects
1169         Set<CdmBase> relatedObjects = deleteResult.getRelatedObjects();
1170
1171         for (CdmBase relatedObject : relatedObjects) {
1172             // check for TypeDesignations
1173             if (relatedObject.isInstanceOf(SpecimenTypeDesignation.class)) {
1174                 SpecimenTypeDesignation designation = HibernateProxyHelper.deproxy(relatedObject, SpecimenTypeDesignation.class);
1175                 designation.setTypeSpecimen(null);
1176                 List<TaxonName> typifiedNames = new ArrayList<>();
1177                 typifiedNames.addAll(designation.getTypifiedNames());
1178                 for (TaxonName taxonName : typifiedNames) {
1179                     taxonName.removeTypeDesignation(designation);
1180                 }
1181             }
1182             // delete IndividualsAssociation
1183             if (relatedObject.isInstanceOf(IndividualsAssociation.class)) {
1184                 IndividualsAssociation association = HibernateProxyHelper.deproxy(relatedObject, IndividualsAssociation.class);
1185                 association.setAssociatedSpecimenOrObservation(null);
1186                 association.getInDescription().removeElement(association);
1187             }
1188             // check for "described specimen" (deprecated)
1189             if (relatedObject.isInstanceOf(TaxonDescription.class)) {
1190                 TaxonDescription description = HibernateProxyHelper.deproxy(relatedObject, TaxonDescription.class);
1191                 description.setDescribedSpecimenOrObservation(null);
1192             }
1193             // check for specimen description
1194             if (relatedObject.isInstanceOf(SpecimenDescription.class)) {
1195                 SpecimenDescription specimenDescription = HibernateProxyHelper.deproxy(relatedObject, SpecimenDescription.class);
1196                 specimenDescription.setDescribedSpecimenOrObservation(null);
1197                 // check if description is a description of the given specimen
1198                 if (specimen.getDescriptions().contains(specimenDescription)) {
1199                     specimen.removeDescription(specimenDescription);
1200                 }
1201                 DeleteResult descriptionDelete = descriptionService.isDeletable(specimenDescription.getUuid(), null);
1202                 if (descriptionDelete.isOk()){
1203                     deleteResult.includeResult(descriptionService.delete(specimenDescription));
1204                 }
1205             }
1206             // check for amplification
1207             if (relatedObject.isInstanceOf(AmplificationResult.class)) {
1208                 AmplificationResult amplificationResult = HibernateProxyHelper.deproxy(relatedObject, AmplificationResult.class);
1209                 amplificationResult.getDnaSample().removeAmplificationResult(amplificationResult);
1210             }
1211             // check for sequence
1212             if (relatedObject.isInstanceOf(Sequence.class)) {
1213                 Sequence sequence = HibernateProxyHelper.deproxy(relatedObject, Sequence.class);
1214                 sequence.getDnaSample().removeSequence(sequence);
1215             }
1216             // check for children and parents (derivation events)
1217             if (relatedObject.isInstanceOf(DerivationEvent.class)) {
1218                 DerivationEvent derivationEvent = HibernateProxyHelper.deproxy(relatedObject, DerivationEvent.class);
1219                 // parent derivation event (derivedFrom)
1220                 if (derivationEvent.getDerivatives().contains(specimen) && specimen.isInstanceOf(DerivedUnit.class)) {
1221                     derivationEvent.removeDerivative(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class));
1222                     if (derivationEvent.getDerivatives().isEmpty()) {
1223                         Set<SpecimenOrObservationBase> originals = new HashSet<>();
1224                         originals.addAll(derivationEvent.getOriginals());
1225                         for (SpecimenOrObservationBase specimenOrObservationBase : originals) {
1226                             specimenOrObservationBase.removeDerivationEvent(derivationEvent);
1227                             deleteResult.addUpdatedObject(specimenOrObservationBase);
1228                         }
1229                         // if derivationEvent has no derivates anymore, delete it
1230                         deleteResult.includeResult(eventService.delete(derivationEvent));
1231                     }
1232                 }
1233                 else{
1234                     //child derivation events should not occur since we delete the hierarchy from bottom to top
1235                 }
1236             }
1237         }
1238         if (specimen instanceof FieldUnit){
1239             FieldUnit fieldUnit = HibernateProxyHelper.deproxy(specimen, FieldUnit.class);
1240             GatheringEvent event = fieldUnit.getGatheringEvent();
1241             fieldUnit.setGatheringEvent(null);
1242             if (event != null){
1243                 DeleteResult result = eventService.isDeletable(event.getUuid(), null);
1244                 if (result.isOk()){
1245                     deleteResult.includeResult( eventService.delete(event));
1246                 }
1247             }
1248
1249         }
1250         deleteResult.includeResult(delete(specimen));
1251
1252         return deleteResult;
1253     }
1254
1255     @Override
1256     public Collection<IndividualsAssociation> listIndividualsAssociations(SpecimenOrObservationBase<?> specimen, Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1257         return dao.listIndividualsAssociations(specimen, limit, start, orderHints, propertyPaths);
1258     }
1259
1260     @Override
1261     public Collection<TaxonBase<?>> listAssociatedTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished,
1262             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1263
1264         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1265
1266         //individuals associations
1267         associatedTaxa.addAll(listIndividualsAssociationTaxa(specimen, includeUnpublished, limit, start, orderHints, propertyPaths));
1268         //type designation
1269         if(specimen.isInstanceOf(DerivedUnit.class)){
1270             associatedTaxa.addAll(listTypeDesignationTaxa(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class),
1271                   includeUnpublished, limit, start, orderHints, propertyPaths));
1272         }
1273         //determinations
1274         associatedTaxa.addAll(listDeterminedTaxa(specimen, includeUnpublished, limit, start, orderHints, propertyPaths));
1275
1276         return associatedTaxa;
1277     }
1278
1279     @Override
1280     public Collection<TaxonBase<?>> listDeterminedTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished,
1281             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1282
1283         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1284         for (DeterminationEvent determinationEvent : listDeterminationEvents(specimen, limit, start, orderHints, propertyPaths)) {
1285             if(determinationEvent.getIdentifiedUnit().equals(specimen)){
1286                 if(determinationEvent.getTaxon()!=null){
1287                     associatedTaxa.add(taxonService.load(determinationEvent.getTaxon().getUuid(), includeUnpublished, propertyPaths));
1288                 }
1289                 if(determinationEvent.getTaxonName()!=null){
1290                     Collection<TaxonBase> taxonBases = determinationEvent.getTaxonName().getTaxonBases();
1291                     for (TaxonBase taxonBase : taxonBases) {
1292                         associatedTaxa.add(taxonService.load(taxonBase.getUuid(), includeUnpublished, propertyPaths));
1293                     }
1294                 }
1295             }
1296         }
1297         return associatedTaxa;
1298     }
1299
1300     /**
1301      * {@inheritDoc}
1302      */
1303     @Override
1304     public Collection<TaxonBase<?>> listTypeDesignationTaxa(DerivedUnit specimen, Integer limit, Integer start,
1305             List<OrderHint> orderHints, List<String> propertyPaths) {
1306         return listTypeDesignationTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1307     }
1308     @Override
1309     public Collection<TaxonBase<?>> listTypeDesignationTaxa(DerivedUnit specimen, boolean includeUnpublished,
1310             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1311         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1312         for (SpecimenTypeDesignation typeDesignation : listTypeDesignations(specimen, limit, start, orderHints, propertyPaths)) {
1313             if(typeDesignation.getTypeSpecimen().equals(specimen)){
1314                 Set<TaxonName> typifiedNames = typeDesignation.getTypifiedNames();
1315                 for (TaxonName taxonName : typifiedNames) {
1316                     Set<Taxon> taxa = taxonName.getTaxa();
1317                     for (Taxon taxon : taxa) {
1318                         associatedTaxa.add(taxonService.load(taxon.getUuid(), includeUnpublished, propertyPaths));
1319                     }
1320                 }
1321             }
1322         }
1323         return associatedTaxa;
1324     }
1325
1326     /**
1327      * {@inheritDoc}
1328      */
1329     @Override
1330     public Collection<TaxonBase<?>> listIndividualsAssociationTaxa(SpecimenOrObservationBase<?> specimen, Integer limit,
1331             Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1332         return listIndividualsAssociationTaxa(specimen, INCLUDE_UNPUBLISHED, limit, start, orderHints, propertyPaths);
1333     }
1334
1335     @Override
1336     public Collection<TaxonBase<?>> listIndividualsAssociationTaxa(SpecimenOrObservationBase<?> specimen, boolean includeUnpublished,
1337             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1338         Collection<TaxonBase<?>> associatedTaxa = new HashSet<>();
1339         for (IndividualsAssociation individualsAssociation : listIndividualsAssociations(specimen, null, null, null, null)) {
1340             if(individualsAssociation.getInDescription().isInstanceOf(TaxonDescription.class)){
1341                 TaxonDescription taxonDescription = HibernateProxyHelper.deproxy(individualsAssociation.getInDescription(), TaxonDescription.class);
1342                 if(taxonDescription.getTaxon()!=null){
1343                     associatedTaxa.add(taxonService.load(taxonDescription.getTaxon().getUuid(), includeUnpublished, propertyPaths));
1344                 }
1345             }
1346         }
1347         return associatedTaxa;
1348     }
1349
1350     @Override
1351     public Collection<DeterminationEvent> listDeterminationEvents(SpecimenOrObservationBase<?> specimen,
1352             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1353         return dao.listDeterminationEvents(specimen, limit, start, orderHints, propertyPaths);
1354     }
1355
1356     @Override
1357     public Map<DerivedUnit, Collection<SpecimenTypeDesignation>> listTypeDesignations(
1358             Collection<DerivedUnit> specimens, Integer limit, Integer start,
1359             List<OrderHint> orderHints, List<String> propertyPaths) {
1360         Map<DerivedUnit, Collection<SpecimenTypeDesignation>> typeDesignationMap = new HashMap<>();
1361         for (DerivedUnit specimen : specimens) {
1362             Collection<SpecimenTypeDesignation> typeDesignations = listTypeDesignations(specimen, limit, start, orderHints, propertyPaths);
1363             typeDesignationMap.put(specimen, typeDesignations);
1364         }
1365         return typeDesignationMap;
1366     }
1367
1368     @Override
1369     public Collection<SpecimenTypeDesignation> listTypeDesignations(DerivedUnit specimen,
1370             Integer limit, Integer start, List<OrderHint> orderHints, List<String> propertyPaths) {
1371         return dao.listTypeDesignations(specimen, limit, start, orderHints, propertyPaths);
1372     }
1373
1374     @Override
1375     public Collection<DescriptionBase<?>> listDescriptionsWithDescriptionSpecimen(
1376             SpecimenOrObservationBase<?> specimen, Integer limit, Integer start, List<OrderHint> orderHints,
1377             List<String> propertyPaths) {
1378         return dao.listDescriptionsWithDescriptionSpecimen(specimen, limit, start, orderHints, propertyPaths);
1379     }
1380
1381     @Override
1382     public Collection<DescriptionElementBase> getCharacterDataForSpecimen(UUID specimenUuid) {
1383         SpecimenOrObservationBase<?> specimen = load(specimenUuid);
1384         if (specimen != null) {
1385             return specimen.characterData();
1386         }
1387         else{
1388             throw new DataRetrievalFailureException("Specimen with the given uuid not found in the data base");
1389         }
1390     }
1391
1392     @Override
1393     public long countByTitle(IIdentifiableEntityServiceConfigurator<SpecimenOrObservationBase> config){
1394
1395         if (config instanceof FindOccurrencesConfigurator) {
1396             FindOccurrencesConfigurator occurrenceConfig = (FindOccurrencesConfigurator) config;
1397             Taxon taxon = null;
1398             if(occurrenceConfig.getAssociatedTaxonUuid()!=null){
1399                 TaxonBase<?> taxonBase = taxonService.load(occurrenceConfig.getAssociatedTaxonUuid());
1400                 if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1401                     taxon = HibernateProxyHelper.deproxy(taxonBase, Taxon.class);
1402                 }
1403             }
1404             TaxonName taxonName = null;
1405             if(occurrenceConfig.getAssociatedTaxonNameUuid()!=null){
1406                 taxonName = nameService.load(occurrenceConfig.getAssociatedTaxonNameUuid());
1407             }
1408             /*TODO: #6484 Neither isRetrieveIndirectlyAssociatedSpecimens() nor the AssignmentStatus
1409              * is currently reflected in the HQL query. So using these in the count method will
1410              * significantly slow down this method as we have to retrieve the entities instead of
1411              * just the amount.
1412              */
1413             if(occurrenceConfig.isRetrieveIndirectlyAssociatedSpecimens() || !occurrenceConfig.getAssignmentStatus().equals(AssignmentStatus.ALL_SPECIMENS)){
1414                 List<SpecimenOrObservationBase> occurrences = new ArrayList<>();
1415                 List<SpecimenOrObservationBase> sobs = dao.findOccurrences(occurrenceConfig.getClazz(),
1416                         occurrenceConfig.getTitleSearchStringSqlized(), occurrenceConfig.getSignificantIdentifier(),
1417                         occurrenceConfig.getSpecimenType(), taxon, taxonName, occurrenceConfig.getMatchMode(),
1418                         occurrenceConfig.isIncludeUnpublished(), null, null,
1419                         occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1420                 occurrences.addAll(sobs);
1421                 occurrences = filterOccurencesByAssignmentAndHierarchy(occurrenceConfig, occurrences, taxon, taxonName);
1422                 return occurrences.size();
1423             }
1424
1425             return dao.countOccurrences(occurrenceConfig.getClazz(),
1426                     occurrenceConfig.getTitleSearchStringSqlized(), occurrenceConfig.getSignificantIdentifier(),
1427                     occurrenceConfig.getSpecimenType(), taxon, taxonName, occurrenceConfig.getMatchMode(),
1428                     occurrenceConfig.isIncludeUnpublished(), null, null,
1429                     occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1430         }
1431         else{
1432             return super.countByTitle(config);
1433         }
1434     }
1435
1436     @Override
1437     public Pager<UuidAndTitleCache<SpecimenOrObservationBase>> findByTitleUuidAndTitleCache(
1438             FindOccurrencesConfigurator config){
1439
1440         List<UuidAndTitleCache<SpecimenOrObservationBase>> occurrences = new ArrayList<>();
1441         Taxon taxon = null;
1442         if(config.getAssociatedTaxonUuid()!=null){
1443             TaxonBase<?> taxonBase = taxonService.load(config.getAssociatedTaxonUuid());
1444             if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1445                 taxon = CdmBase.deproxy(taxonBase, Taxon.class);
1446             }
1447         }
1448         TaxonName taxonName = null;
1449         if(config.getAssociatedTaxonNameUuid()!=null){
1450             taxonName = nameService.load(config.getAssociatedTaxonNameUuid());
1451         }
1452         occurrences.addAll(dao.findOccurrencesUuidAndTitleCache(config.getClazz(),
1453                 config.getTitleSearchString(), config.getSignificantIdentifier(),
1454                 config.getSpecimenType(), taxon, taxonName, config.getMatchMode(),
1455                 config.isIncludeUnpublished(),
1456                 null, null, config.getOrderHints()));
1457         long count = Integer.valueOf(occurrences.size()).longValue();
1458         return new DefaultPagerImpl<>(config.getPageNumber(), count, config.getPageSize(), occurrences);
1459     }
1460
1461     @Override
1462     public List<DerivedUnitDTO> findByTitleDerivedUnitDTO(FindOccurrencesConfigurator config) {
1463
1464         Taxon taxon = null;
1465         if(config.getAssociatedTaxonUuid()!=null){
1466             TaxonBase<?> taxonBase = taxonService.load(config.getAssociatedTaxonUuid());
1467             if(taxonBase != null && taxonBase.isInstanceOf(Taxon.class)){
1468                 taxon = CdmBase.deproxy(taxonBase, Taxon.class);
1469             }
1470         }
1471         TaxonName taxonName = null;
1472         if(config.getAssociatedTaxonNameUuid()!=null){
1473             taxonName = nameService.load(config.getAssociatedTaxonNameUuid());
1474         }
1475         List<DerivedUnit> occurrences = new ArrayList<>();
1476         occurrences.addAll(dao.findOccurrences(DerivedUnit.class,
1477                 config.getTitleSearchString(), config.getSignificantIdentifier(),
1478                 config.getSpecimenType(), taxon, taxonName, config.getMatchMode(),
1479                 config.isIncludeUnpublished(), null, null,
1480                 config.getOrderHints(), null));
1481
1482         List<DerivedUnitDTO> dtos = new ArrayList<>();
1483         occurrences.forEach(derivedUnit->dtos.add(assembleDerivedUnitDTO(derivedUnit)));
1484         return dtos;
1485     }
1486
1487     @Override
1488     public <S extends SpecimenOrObservationBase> Pager<S> findByTitle(
1489             IIdentifiableEntityServiceConfigurator<S> config) {
1490
1491         if (config instanceof FindOccurrencesConfigurator) {
1492             FindOccurrencesConfigurator occurrenceConfig = (FindOccurrencesConfigurator) config;
1493             List<SpecimenOrObservationBase> occurrences = new ArrayList<>();
1494             Taxon taxon = null;
1495             if(occurrenceConfig.getAssociatedTaxonUuid()!=null){
1496                 TaxonBase<?> taxonBase = taxonService.load(occurrenceConfig.getAssociatedTaxonUuid());
1497                 if(taxonBase.isInstanceOf(Taxon.class)){
1498                     taxon = HibernateProxyHelper.deproxy(taxonBase, Taxon.class);
1499                 }
1500             }
1501             TaxonName taxonName = null;
1502             if(occurrenceConfig.getAssociatedTaxonNameUuid()!=null){
1503                 taxonName = nameService.load(occurrenceConfig.getAssociatedTaxonNameUuid());
1504             }
1505             List<? extends SpecimenOrObservationBase> foundOccurrences = dao.findOccurrences(
1506                     occurrenceConfig.getClazz(),
1507                     occurrenceConfig.getTitleSearchString(), occurrenceConfig.getSignificantIdentifier(),
1508                     occurrenceConfig.getSpecimenType(), taxon, taxonName,
1509                     occurrenceConfig.getMatchMode(), occurrenceConfig.isIncludeUnpublished(),
1510                     null, null, occurrenceConfig.getOrderHints(), occurrenceConfig.getPropertyPaths());
1511             occurrences.addAll(foundOccurrences);
1512             occurrences = filterOccurencesByAssignmentAndHierarchy(occurrenceConfig, occurrences, taxon, taxonName);
1513
1514             long count = Integer.valueOf(occurrences.size()).longValue();
1515             return new DefaultPagerImpl<>(config.getPageNumber(), count, config.getPageSize(), (List<S>)occurrences);
1516         }
1517         return super.findByTitle(config);
1518     }
1519
1520     private List<SpecimenOrObservationBase> filterOccurencesByAssignmentAndHierarchy(
1521             FindOccurrencesConfigurator occurrenceConfig, List<SpecimenOrObservationBase> occurrences, Taxon taxon,
1522             TaxonName taxonName) {
1523         //filter out (un-)assigned specimens
1524         if(taxon==null && taxonName==null){
1525             AssignmentStatus assignmentStatus = occurrenceConfig.getAssignmentStatus();
1526             List<SpecimenOrObservationBase> specimenWithAssociations = new ArrayList<>();
1527             if(!assignmentStatus.equals(AssignmentStatus.ALL_SPECIMENS)){
1528                 for (SpecimenOrObservationBase<?> specimenOrObservationBase : occurrences) {
1529                     boolean includeUnpublished = true;  //TODO not sure if this is correct, maybe we have to propagate publish flag to higher methods.
1530                     Collection<TaxonBase<?>> associatedTaxa = listAssociatedTaxa(specimenOrObservationBase,
1531                             includeUnpublished, null, null, null, null);
1532                     if(!associatedTaxa.isEmpty()){
1533                         specimenWithAssociations.add(specimenOrObservationBase);
1534                     }
1535                 }
1536             }
1537             if(assignmentStatus.equals(AssignmentStatus.UNASSIGNED_SPECIMENS)){
1538                 occurrences.removeAll(specimenWithAssociations);
1539             }
1540             if(assignmentStatus.equals(AssignmentStatus.ASSIGNED_SPECIMENS)){
1541                 occurrences = new ArrayList<>(specimenWithAssociations);
1542             }
1543         }
1544         // indirectly associated specimens
1545         if(occurrenceConfig.isRetrieveIndirectlyAssociatedSpecimens()){
1546             List<SpecimenOrObservationBase> indirectlyAssociatedOccurrences = new ArrayList<>(occurrences);
1547             for (SpecimenOrObservationBase<?> specimen : occurrences) {
1548                 List<SpecimenOrObservationBase<?>> allHierarchyDerivates = getAllHierarchyDerivatives(specimen);
1549                 for (SpecimenOrObservationBase<?> specimenOrObservationBase : allHierarchyDerivates) {
1550                     if(!occurrences.contains(specimenOrObservationBase)){
1551                         indirectlyAssociatedOccurrences.add(specimenOrObservationBase);
1552                     }
1553                 }
1554             }
1555             occurrences = indirectlyAssociatedOccurrences;
1556         }
1557         return occurrences;
1558     }
1559
1560     @Override
1561     public List<SpecimenOrObservationBase<?>> getAllHierarchyDerivatives(SpecimenOrObservationBase<?> specimen){
1562         List<SpecimenOrObservationBase<?>> allHierarchyDerivatives = new ArrayList<>();
1563         Collection<FieldUnit> fieldUnits = findFieldUnits(specimen.getUuid(), null);
1564         if(fieldUnits.isEmpty()){
1565             allHierarchyDerivatives.add(specimen);
1566             allHierarchyDerivatives.addAll(getAllChildDerivatives(specimen));
1567         }
1568         else{
1569             for (FieldUnit fieldUnit : fieldUnits) {
1570                 allHierarchyDerivatives.add(fieldUnit);
1571                 allHierarchyDerivatives.addAll(getAllChildDerivatives(fieldUnit));
1572             }
1573         }
1574         return allHierarchyDerivatives;
1575     }
1576
1577     @Override
1578     public List<DerivedUnit> getAllChildDerivatives(UUID specimenUuid){
1579         return getAllChildDerivatives(load(specimenUuid));
1580     }
1581
1582     @Override
1583     public List<DerivedUnit> getAllChildDerivatives(SpecimenOrObservationBase<?> specimen){
1584         if (specimen == null){
1585             return null;
1586         }
1587         List<DerivedUnit> childDerivate = new ArrayList<>();
1588         Set<DerivationEvent> derivationEvents = specimen.getDerivationEvents();
1589         for (DerivationEvent derivationEvent : derivationEvents) {
1590             Set<DerivedUnit> derivatives = derivationEvent.getDerivatives();
1591             for (DerivedUnit derivedUnit : derivatives) {
1592                 childDerivate.add(derivedUnit);
1593                 childDerivate.addAll(getAllChildDerivatives(derivedUnit.getUuid()));
1594             }
1595         }
1596         return childDerivate;
1597     }
1598
1599     @Override
1600     public long countOccurrences(IIdentifiableEntityServiceConfigurator<SpecimenOrObservationBase> config){
1601         return countByTitle(config);
1602     }
1603
1604     @Override
1605     public List<FieldUnit> findFieldUnitsForGatheringEvent(UUID gatheringEventUuid) {
1606         return dao.findFieldUnitsForGatheringEvent(gatheringEventUuid, null, null, null, null);
1607     }
1608
1609     @Override
1610     public List<Point> findPointsForFieldUnitList(List<UUID> fieldUnitUuids) {
1611         return dao.findPointsForFieldUnitList(fieldUnitUuids);
1612     }
1613 }