cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/DescriptiveDataSetService.java

   1 package eu.etaxonomy.cdm.api.service;
   2
   3 import java.util.ArrayList;
   4 import java.util.Arrays;
   5 import java.util.Collection;
   6 import java.util.HashMap;
   7 import java.util.HashSet;
   8 import java.util.List;
   9 import java.util.Map;
  10 import java.util.Map.Entry;
  11 import java.util.Optional;
  12 import java.util.Set;
  13 import java.util.UUID;
  14 import java.util.stream.Collectors;
  15
  16 import org.apache.log4j.Logger;
  17 import org.springframework.beans.factory.annotation.Autowired;
  18 import org.springframework.stereotype.Service;
  19 import org.springframework.transaction.annotation.Transactional;
  20
  21 import eu.etaxonomy.cdm.api.service.UpdateResult.Status;
  22 import eu.etaxonomy.cdm.api.service.config.DescriptionAggregationConfiguration;
  23 import eu.etaxonomy.cdm.api.service.dto.RowWrapperDTO;
  24 import eu.etaxonomy.cdm.api.service.dto.SpecimenRowWrapperDTO;
  25 import eu.etaxonomy.cdm.api.service.dto.TaxonRowWrapperDTO;
  26 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  27 import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
  28 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  29 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  30 import eu.etaxonomy.cdm.model.common.Language;
  31 import eu.etaxonomy.cdm.model.description.CategoricalData;
  32 import eu.etaxonomy.cdm.model.description.Character;
  33 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  34 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  35 import eu.etaxonomy.cdm.model.description.DescriptionType;
  36 import eu.etaxonomy.cdm.model.description.DescriptiveDataSet;
  37 import eu.etaxonomy.cdm.model.description.DescriptiveSystemRole;
  38 import eu.etaxonomy.cdm.model.description.Feature;
  39 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  40 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  41 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  42 import eu.etaxonomy.cdm.model.description.StateData;
  43 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  44 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  45 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  46 import eu.etaxonomy.cdm.model.description.TextData;
  47 import eu.etaxonomy.cdm.model.location.NamedArea;
  48 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  49 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  50 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  51 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  52 import eu.etaxonomy.cdm.model.taxon.Classification;
  53 import eu.etaxonomy.cdm.model.taxon.Taxon;
  54 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  55 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  56 import eu.etaxonomy.cdm.persistence.dao.description.IDescriptiveDataSetDao;
  57 import eu.etaxonomy.cdm.persistence.dao.term.IDefinedTermDao;
  58 import eu.etaxonomy.cdm.persistence.dto.SpecimenNodeWrapper;
  59 import eu.etaxonomy.cdm.persistence.dto.TaxonNodeDto;
  60 import eu.etaxonomy.cdm.persistence.dto.TermDto;
  61 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  62 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
  63
  64 @Service
  65 @Transactional(readOnly=true)
  66 public class DescriptiveDataSetService
  67         extends IdentifiableServiceBase<DescriptiveDataSet, IDescriptiveDataSetDao>
  68         implements IDescriptiveDataSetService {
  69
  70     private static Logger logger = Logger.getLogger(DescriptiveDataSetService.class);
  71
  72     @Autowired
  73     private IOccurrenceService occurrenceService;
  74
  75     @Autowired
  76     private ITaxonService taxonService;
  77
  78     @Autowired
  79     private IDefinedTermDao termDao;
  80
  81     @Autowired
  82     private IDescriptionService descriptionService;
  83
  84     @Autowired
  85     private ITaxonNodeService taxonNodeService;
  86
  87         @Override
  88         @Autowired
  89         protected void setDao(IDescriptiveDataSetDao dao) {
  90                 this.dao = dao;
  91         }
  92
  93         @Override
  94     public Map<DescriptionBase, Set<DescriptionElementBase>> getDescriptionElements(DescriptiveDataSet descriptiveDataSet, Set<Feature> features, Integer pageSize,     Integer pageNumber,
  95                         List<String> propertyPaths) {
  96                 return dao.getDescriptionElements(descriptiveDataSet, features, pageSize, pageNumber, propertyPaths);
  97         }
  98
  99         @Override
 100         public <T extends DescriptionElementBase> Map<UuidAndTitleCache, Map<UUID, Set<T>>> getTaxonFeatureDescriptionElementMap(
 101                         Class<T> clazz, UUID descriptiveDataSetUuid, DescriptiveSystemRole role) {
 102                 return dao.getTaxonFeatureDescriptionElementMap(clazz, descriptiveDataSetUuid, role);
 103         }
 104
 105         @Override
 106     public List<UuidAndTitleCache<DescriptiveDataSet>> getDescriptiveDataSetUuidAndTitleCache(Integer limitOfInitialElements, String pattern) {
 107         return dao.getDescriptiveDataSetUuidAndTitleCache( limitOfInitialElements, pattern);
 108     }
 109
 110         @Override
 111         public ArrayList<RowWrapperDTO> getRowWrapper(UUID descriptiveDataSetUuid, IProgressMonitor monitor) {
 112             DescriptiveDataSet descriptiveDataSet = load(descriptiveDataSetUuid);
 113             monitor.beginTask("Load row wrapper", descriptiveDataSet.getDescriptions().size());
 114             ArrayList<RowWrapperDTO> wrappers = new ArrayList<>();
 115             Set<DescriptionBase> descriptions = descriptiveDataSet.getDescriptions();
 116             for (DescriptionBase description : descriptions) {
 117             if(monitor.isCanceled()){
 118                 return new ArrayList<>();
 119             }
 120             RowWrapperDTO rowWrapper = null;
 121             // only viable descriptions are aggregated, literature or default descriptions
 122             if(HibernateProxyHelper.isInstanceOf(description, TaxonDescription.class) &&
 123                     (description.getTypes().contains(DescriptionType.AGGREGATED)
 124                             || description.getTypes().contains(DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION)
 125                             || description.getTypes().contains(DescriptionType.SECONDARY_DATA)
 126                             )){
 127                 rowWrapper = createTaxonRowWrapper(description.getUuid(), descriptiveDataSet.getUuid());
 128             }
 129             else if (HibernateProxyHelper.isInstanceOf(description, SpecimenDescription.class)){
 130                 rowWrapper = createSpecimenRowWrapper(HibernateProxyHelper.deproxy(description, SpecimenDescription.class), descriptiveDataSetUuid);
 131             }
 132             if(rowWrapper!=null){
 133                 wrappers.add(rowWrapper);
 134             }
 135             monitor.worked(1);
 136         }
 137             return wrappers;
 138         }
 139
 140     @Override
 141     public Collection<SpecimenNodeWrapper> loadSpecimens(DescriptiveDataSet descriptiveDataSet){
 142         List<UUID> filteredNodes = findFilteredTaxonNodes(descriptiveDataSet);
 143         return occurrenceService.listUuidAndTitleCacheByAssociatedTaxon(filteredNodes, null, null);
 144     }
 145
 146     @Override
 147     public List<UUID> findFilteredTaxonNodes(DescriptiveDataSet descriptiveDataSet){
 148         TaxonNodeFilter filter = TaxonNodeFilter.NewRankInstance(descriptiveDataSet.getMinRank(), descriptiveDataSet.getMaxRank());
 149         descriptiveDataSet.getGeoFilter().forEach(area -> filter.orArea(area.getUuid()));
 150         descriptiveDataSet.getTaxonSubtreeFilter().forEach(node -> filter.orSubtree(node));
 151         filter.setIncludeUnpublished(true);
 152
 153         return taxonNodeService.uuidList(filter);
 154     }
 155
 156     @Override
 157     public List<TaxonNode> loadFilteredTaxonNodes(DescriptiveDataSet descriptiveDataSet, List<String> propertyPaths){
 158         return taxonNodeService.load(findFilteredTaxonNodes(descriptiveDataSet), propertyPaths);
 159     }
 160
 161     private TaxonNode findTaxonNodeForDescription(SpecimenDescription description, DescriptiveDataSet descriptiveDataSet){
 162         SpecimenOrObservationBase specimen = description.getDescribedSpecimenOrObservation();
 163         TaxonNode taxonNode = null;
 164         //get taxon node
 165
 166         Set<IndividualsAssociation> associations = (Set<IndividualsAssociation>) descriptiveDataSet.getDescriptions()
 167                 .stream()
 168                 .flatMap(desc->desc.getElements().stream())// put all description element in one stream
 169                 .filter(element->element instanceof IndividualsAssociation)
 170                 .map(ia->(IndividualsAssociation)ia)
 171                 .collect(Collectors.toSet());
 172         Classification classification = descriptiveDataSet.getTaxonSubtreeFilter().iterator().next().getClassification();
 173         for (IndividualsAssociation individualsAssociation : associations) {
 174             if(individualsAssociation.getAssociatedSpecimenOrObservation().equals(specimen)){
 175                 return ((TaxonDescription) individualsAssociation.getInDescription()).getTaxon().getTaxonNode(classification);
 176             }
 177         }
 178         return null;
 179     }
 180
 181     private TaxonNode findTaxonNodeForSpecimen(TaxonNode taxonNode, SpecimenOrObservationBase<?> specimen){
 182         Collection<SpecimenNodeWrapper> nodeWrapper = occurrenceService.listUuidAndTitleCacheByAssociatedTaxon(Arrays.asList(taxonNode.getUuid()), null, null);
 183         for (SpecimenNodeWrapper specimenNodeWrapper : nodeWrapper) {
 184             if(specimenNodeWrapper.getUuidAndTitleCache().getId().equals(specimen.getId())){
 185                 return taxonNode;
 186             }
 187         }
 188         return null;
 189     }
 190
 191     @Override
 192     public TaxonRowWrapperDTO createTaxonRowWrapper(UUID taxonDescriptionUuid, UUID descriptiveDataSetUuid) {
 193         TaxonNode taxonNode = null;
 194         Classification classification = null;
 195         TaxonDescription description = (TaxonDescription) descriptionService.load(taxonDescriptionUuid,
 196                 Arrays.asList("taxon", "descriptionElements", "descriptionElements.feature"));
 197         DescriptiveDataSet descriptiveDataSet = dao.load(descriptiveDataSetUuid, null);
 198         Optional<TaxonNode> first = descriptiveDataSet.getTaxonSubtreeFilter().stream()
 199                 .filter(node->node.getClassification()!=null).findFirst();
 200         Optional<Classification> classificationOptional = first.map(node->node.getClassification());
 201         if(classificationOptional.isPresent()){
 202             classification = classificationOptional.get();
 203             Taxon taxon = (Taxon) taxonService.load(description.getTaxon().getId(), Arrays.asList("taxonNodes", "taxonNodes.classification"));
 204             taxonNode = taxon.getTaxonNode(classification);
 205         }
 206         return new TaxonRowWrapperDTO(description, new TaxonNodeDto(taxonNode));
 207     }
 208
 209     @Override
 210     @Transactional(readOnly=false)
 211     public UpdateResult addRowWrapperToDataset(Collection<SpecimenNodeWrapper> wrappers, UUID datasetUuid){
 212         UpdateResult result = new UpdateResult();
 213         DescriptiveDataSet dataSet = load(datasetUuid);
 214         result.setCdmEntity(dataSet);
 215         for (SpecimenNodeWrapper wrapper : wrappers) {
 216             UUID taxonDescriptionUuid = wrapper.getTaxonDescriptionUuid();
 217             TaxonDescription taxonDescription = null;
 218             if(taxonDescriptionUuid!=null){
 219                 taxonDescription = (TaxonDescription) descriptionService.load(taxonDescriptionUuid);
 220             }
 221             if(taxonDescription==null){
 222                 Optional<TaxonDescription> associationDescriptionOptional = wrapper.getTaxonNode().getTaxon().getDescriptions().stream()
 223                         .filter(desc->desc.getTypes().contains(DescriptionType.INDIVIDUALS_ASSOCIATION))
 224                         .findFirst();
 225                 Taxon taxon = wrapper.getTaxonNode().getTaxon();
 226                 if(!associationDescriptionOptional.isPresent()){
 227                     taxonDescription = TaxonDescription.NewInstance(taxon);
 228                 }
 229                 else{
 230                     taxonDescription = associationDescriptionOptional.get();
 231                 }
 232
 233                 SpecimenOrObservationBase specimen = occurrenceService.load(wrapper.getUuidAndTitleCache().getUuid());
 234                 IndividualsAssociation association = IndividualsAssociation.NewInstance(specimen);
 235                 taxonDescription.addElement(association);
 236                 taxonService.saveOrUpdate(taxon);
 237                 result.addUpdatedObject(taxon);
 238             }
 239             SpecimenDescription specimenDescription = findSpecimenDescription(datasetUuid, wrapper.getUuidAndTitleCache().getUuid(), true);
 240             SpecimenRowWrapperDTO rowWrapper = createSpecimenRowWrapper(specimenDescription, wrapper.getTaxonNode().getUuid(), datasetUuid);
 241             if(rowWrapper==null){
 242                 result.addException(new IllegalArgumentException("Could not create wrapper for "+specimenDescription));
 243                 continue;
 244             }
 245             //add specimen description to data set
 246             rowWrapper.getDescription().addDescriptiveDataSet(dataSet);
 247             //add taxon description with IndividualsAssociation to the specimen to data set
 248             taxonDescription.addDescriptiveDataSet(dataSet);
 249
 250             result.addUpdatedObject(rowWrapper.getDescription());
 251             result.addUpdatedObject(taxonDescription);
 252         }
 253         saveOrUpdate(dataSet);
 254         return result;
 255     }
 256
 257     private SpecimenRowWrapperDTO createSpecimenRowWrapper(SpecimenDescription description, UUID taxonNodeUuid,
 258             UUID datasetUuid) {
 259         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid);
 260         DescriptiveDataSet descriptiveDataSet = load(datasetUuid);
 261         SpecimenOrObservationBase specimen = description.getDescribedSpecimenOrObservation();
 262         //supplemental information
 263         if(taxonNode==null){
 264             taxonNode = findTaxonNodeForDescription(description, descriptiveDataSet);
 265         }
 266         FieldUnit fieldUnit = null;
 267         String identifier = null;
 268         NamedArea country = null;
 269         if(taxonNode==null){
 270             return null;
 271         }
 272         //taxon node was found
 273
 274         //get field unit
 275         Collection<FieldUnit> fieldUnits = occurrenceService.findFieldUnits(specimen.getUuid(),
 276                 Arrays.asList(new String[]{
 277                         "gatheringEvent",
 278                         "gatheringEvent.country"
 279                 }));
 280         if(fieldUnits.size()!=1){
 281             logger.error("More than one or no field unit found for specimen"); //$NON-NLS-1$
 282             return null;
 283         }
 284         else{
 285             fieldUnit = fieldUnits.iterator().next();
 286         }
 287         //get identifier
 288         if(HibernateProxyHelper.isInstanceOf(specimen, DerivedUnit.class)){
 289             identifier = occurrenceService.getMostSignificantIdentifier(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class));
 290         }
 291         //get country
 292         if(fieldUnit!=null && fieldUnit.getGatheringEvent()!=null){
 293             country = fieldUnit.getGatheringEvent().getCountry();
 294         }
 295         //get default taxon description
 296         TaxonDescription defaultTaxonDescription = findTaxonDescriptionByDescriptionType(descriptiveDataSet.getUuid(),
 297                 taxonNode.getUuid(), DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION);
 298         TaxonRowWrapperDTO taxonRowWrapper = defaultTaxonDescription != null
 299                 ? createTaxonRowWrapper(defaultTaxonDescription.getUuid(), descriptiveDataSet.getUuid()) : null;
 300         return new SpecimenRowWrapperDTO(description, new TaxonNodeDto(taxonNode), fieldUnit, identifier, country);
 301     }
 302
 303     @Override
 304     public SpecimenRowWrapperDTO createSpecimenRowWrapper(SpecimenDescription description, UUID descriptiveDataSetUuid){
 305         return createSpecimenRowWrapper(description, null, descriptiveDataSetUuid);
 306         }
 307
 308     @Override
 309     @Transactional(readOnly = false)
 310     public UpdateResult updateCaches(Class<? extends DescriptiveDataSet> clazz, Integer stepSize,
 311             IIdentifiableEntityCacheStrategy<DescriptiveDataSet> cacheStrategy, IProgressMonitor monitor) {
 312         if (clazz == null) {
 313             clazz = DescriptiveDataSet.class;
 314         }
 315         return super.updateCachesImpl(clazz, stepSize, cacheStrategy, monitor);
 316     }
 317
 318     private TaxonDescription findTaxonDescriptionByDescriptionType(DescriptiveDataSet dataSet, Taxon taxon, DescriptionType descriptionType){
 319         Set<DescriptionBase> dataSetDescriptions = dataSet.getDescriptions();
 320         //filter by DEFAULT descriptions
 321         Optional<TaxonDescription> first = taxon.getDescriptions().stream()
 322                 .filter(desc -> desc.getTypes().stream().anyMatch(type -> type.equals(descriptionType)))
 323                 .filter(defaultDescription->dataSetDescriptions.contains(defaultDescription))
 324                 .findFirst();
 325         if(first.isPresent()){
 326             return HibernateProxyHelper.deproxy(descriptionService.load(first.get().getUuid(),
 327                   Arrays.asList("taxon", "descriptionElements", "descriptionElements.feature")), TaxonDescription.class);
 328         }
 329         return null;
 330     }
 331
 332     @Override
 333     public TaxonDescription findTaxonDescriptionByDescriptionType(UUID dataSetUuid, UUID taxonNodeUuid, DescriptionType descriptionType){
 334         DescriptiveDataSet dataSet = load(dataSetUuid);
 335         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid);
 336         return findTaxonDescriptionByDescriptionType(dataSet, taxonNode.getTaxon(), descriptionType);
 337     }
 338
 339     @Override
 340     @Transactional(readOnly=false)
 341     public UpdateResult aggregate(UUID descriptiveDataSetUuid, DescriptionAggregationConfiguration config, IProgressMonitor monitor) {
 342         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 343         Set<DescriptionBase> descriptions = dataSet.getDescriptions();
 344
 345         monitor.beginTask("Aggregate data set", descriptions.size()*2);
 346
 347         UpdateResult result = new UpdateResult();
 348         result.setCdmEntity(dataSet);
 349
 350         // clone specimen descriptions
 351         // create a snapshot of those descriptions that were used to create the aggregated descriptions
 352         // TODO implement when the clones descriptions can be attached to taxon descriptions as sources
 353 //        for (DescriptionBase<?> descriptionBase : descriptions) {
 354 //            if(descriptionBase instanceof SpecimenDescription){
 355 //                SpecimenDescription specimenDescription = (SpecimenDescription)descriptionBase;
 356 //                SpecimenOrObservationBase<?> specimenOrObservation = specimenDescription.getDescribedSpecimenOrObservation();
 357 //                SpecimenDescription clone = (SpecimenDescription) specimenDescription.clone();
 358 //                clone.setDescriptionType(DescriptionType.AggregationClone);
 359 //                specimenOrObservation.addDescription(clone);
 360 //            }
 361 //        }
 362
 363         // sort descriptions by taxa
 364         Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap = new HashMap<>();
 365         for (DescriptionBase descriptionBase : descriptions) {
 366             if(monitor.isCanceled()){
 367                 result.setAbort();
 368                 return result;
 369             }
 370
 371             if(descriptionBase instanceof SpecimenDescription){
 372                 SpecimenDescription specimenDescription = HibernateProxyHelper.deproxy(descriptionBase, SpecimenDescription.class);
 373                 TaxonNode taxonNode = findTaxonNodeForDescription(specimenDescription, dataSet);
 374                 if(taxonNode!=null){
 375                     addDescriptionToTaxonNodeMap(specimenDescription.getUuid(), taxonNode, taxonNodeToSpecimenDescriptionMap);
 376                 }
 377             }
 378             monitor.worked(1);
 379         }
 380         if(config.isRecursiveAggregation()){
 381             propagateDescriptionsToParentNodes(dataSet, taxonNodeToSpecimenDescriptionMap);
 382         }
 383         // aggregate per taxa
 384         for (Entry<TaxonNode, Set<UUID>> entry: taxonNodeToSpecimenDescriptionMap.entrySet()) {
 385             if(monitor.isCanceled()){
 386                 result.setAbort();
 387                 return result;
 388             }
 389             UUID taxonUuid = entry.getKey().getTaxon().getUuid();
 390             Set<UUID> specimenDescriptionUuids = entry.getValue();
 391             result.includeResult(aggregateDescription(taxonUuid , specimenDescriptionUuids, descriptiveDataSetUuid));
 392             monitor.worked(1);
 393         }
 394         monitor.done();
 395         return result;
 396     }
 397
 398     private void addDescriptionToTaxonNodeMap(UUID descriptionUuid, TaxonNode taxonNode, Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap){
 399         Set<UUID> specimenDescriptionUuids = taxonNodeToSpecimenDescriptionMap.get(taxonNode);
 400         if(specimenDescriptionUuids==null){
 401             specimenDescriptionUuids = new HashSet<>();
 402         }
 403         specimenDescriptionUuids.add(descriptionUuid);
 404         taxonNodeToSpecimenDescriptionMap.put(taxonNode, specimenDescriptionUuids);
 405     }
 406
 407     private void propagateDescriptionsToParentNodes(DescriptiveDataSet dataSet, Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap){
 408         Map<TaxonNode, Set<UUID>> parentMap = new HashMap<>();
 409         for (Entry<TaxonNode, Set<UUID>> entry: taxonNodeToSpecimenDescriptionMap.entrySet()) {
 410             Set<UUID> descriptionUuids = entry.getValue();
 411             TaxonNode node = entry.getKey();
 412             TaxonNode parentNode = node.getParent();
 413             while(parentNode!=null && isTaxonNodeInDescriptiveDataSet(parentNode, dataSet)){
 414                 for (UUID uuid : descriptionUuids) {
 415                     addDescriptionToTaxonNodeMap(uuid, node.getParent(), parentMap);
 416                 }
 417                 parentNode = parentNode.getParent();
 418             }
 419         }
 420         // merge parent map
 421         for (Entry<TaxonNode, Set<UUID>> entry: parentMap.entrySet()) {
 422             Set<UUID> descriptionUuids = entry.getValue();
 423             TaxonNode node = entry.getKey();
 424             for (UUID uuid : descriptionUuids) {
 425                 addDescriptionToTaxonNodeMap(uuid, node, taxonNodeToSpecimenDescriptionMap);
 426             }
 427         }
 428     }
 429
 430     private boolean isTaxonNodeInDescriptiveDataSet(TaxonNode taxonNode, DescriptiveDataSet dataSet){
 431         Set<TaxonNode> taxonSubtreeFilter = dataSet.getTaxonSubtreeFilter();
 432         for (TaxonNode datasetNode : taxonSubtreeFilter) {
 433             if(datasetNode.getUuid().equals(taxonNode.getUuid())){
 434                 return true;
 435             }
 436             List<TaxonNode> allChildren = taxonNodeService.loadChildNodesOfTaxonNode(datasetNode, null, true, true, null);
 437             for (TaxonNode childNode : allChildren) {
 438                 if(childNode.getUuid().equals(taxonNode.getUuid())){
 439                     return true;
 440                 }
 441             }
 442         }
 443         return false;
 444     }
 445
 446     @SuppressWarnings("unchecked")
 447     private UpdateResult aggregateDescription(UUID taxonUuid, Set<UUID> descriptionUuids, UUID descriptiveDataSetUuid) {
 448         UpdateResult result = new UpdateResult();
 449
 450         TaxonBase taxonBase = taxonService.load(taxonUuid);
 451         if(!(taxonBase instanceof Taxon)){
 452             result.addException(new ClassCastException("The given taxonUUID does not belong to a taxon"));
 453             result.setError();
 454             return result;
 455         }
 456         Taxon taxon = (Taxon)taxonBase;
 457         List<DescriptionBase> descriptions = descriptionService.load(new ArrayList<>(descriptionUuids), null);
 458         Map<Character, List<DescriptionElementBase>> featureToElementMap = new HashMap<>();
 459
 460         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 461         if(dataSet==null){
 462             result.addException(new IllegalArgumentException("Could not find data set for uuid "+descriptiveDataSetUuid));
 463             result.setAbort();
 464             return result;
 465         }
 466
 467         //extract all character description elements
 468         descriptions.forEach(description->{
 469             description.getElements()
 470             .stream()
 471             //filter out elements that do not have a Character as Feature
 472             .filter(element->HibernateProxyHelper.isInstanceOf(((DescriptionElementBase)element).getFeature(), Character.class))
 473             .forEach(ele->{
 474                 DescriptionElementBase descriptionElement = (DescriptionElementBase)ele;
 475                 List<DescriptionElementBase> list = featureToElementMap.get(descriptionElement.getFeature());
 476                 if(list==null){
 477                     list = new ArrayList<>();
 478                 }
 479                 list.add(descriptionElement);
 480                 featureToElementMap.put(HibernateProxyHelper.deproxy(descriptionElement.getFeature(), Character.class), list);
 481             });
 482         });
 483
 484         // delete all aggregation description of this dataset for this taxon (DescriptionType.AGGREGATED)
 485         List<TaxonDescription> toRemove = dataSet.getDescriptions().stream()
 486         .filter(aggDesc->aggDesc instanceof TaxonDescription)
 487         .filter(desc -> desc.getTypes().stream().anyMatch(type -> type.equals(DescriptionType.AGGREGATED)))
 488         .map(aggDesc->(TaxonDescription)aggDesc)
 489         .collect(Collectors.toList());
 490         for (TaxonDescription taxonDescription : toRemove) {
 491             if(taxon.getDescriptions().contains(taxonDescription)){
 492                 dataSet.removeDescription(taxonDescription);
 493                 taxon.removeDescription(taxonDescription);
 494             }
 495         }
 496
 497         // create new aggregation
 498         TaxonDescription description = TaxonDescription.NewInstance(taxon);
 499         description.setTitleCache("[Aggregation] "+dataSet.getTitleCache(), true);
 500         description.getTypes().add(DescriptionType.AGGREGATED);
 501         IdentifiableSource source = IdentifiableSource.NewInstance(OriginalSourceType.Aggregation);
 502         description.addSource(source);
 503         description.addDescriptiveDataSet(dataSet);
 504
 505         featureToElementMap.forEach((feature, elements)->{
 506             //aggregate categorical data
 507             if(feature.isSupportsCategoricalData()){
 508                 CategoricalData aggregate = CategoricalData.NewInstance(feature);
 509                 elements.stream()
 510                 .filter(element->element instanceof CategoricalData)
 511                 .forEach(categoricalData->((CategoricalData)categoricalData).getStateData()
 512                         .forEach(stateData->aggregate.addStateData((StateData) stateData.clone())));
 513                 description.addElement(aggregate);
 514             }
 515             //aggregate quantitative data
 516             else if(feature.isSupportsQuantitativeData()){
 517                 QuantitativeData aggregate = QuantitativeData.NewInstance(feature);
 518                 elements.stream()
 519                 .filter(element->element instanceof QuantitativeData)
 520                 .forEach(categoricalData->((QuantitativeData)categoricalData).getStatisticalValues()
 521                         .forEach(statisticalValue->aggregate.addStatisticalValue((StatisticalMeasurementValue) statisticalValue.clone())));
 522                 description.addElement(aggregate);
 523             }
 524         });
 525         result.addUpdatedObject(taxon);
 526         result.addUpdatedObject(description);
 527
 528         return result;
 529     }
 530
 531     @Override
 532     @Transactional(readOnly=false)
 533     public DeleteResult removeDescription(UUID descriptionUuid, UUID descriptiveDataSetUuid) {
 534         DeleteResult result = new DeleteResult();
 535         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 536         DescriptionBase descriptionBase = descriptionService.load(descriptionUuid);
 537         if(dataSet==null || descriptionBase==null){
 538             result.setError();
 539         }
 540         else{
 541             boolean success = dataSet.removeDescription(descriptionBase);
 542             result.addDeletedObject(descriptionBase);
 543             // remove taxon description with IndividualsAssociation from data set
 544             if(descriptionBase instanceof SpecimenDescription){
 545                 Set<IndividualsAssociation> associations = (Set<IndividualsAssociation>) dataSet.getDescriptions()
 546                         .stream()
 547                         .flatMap(desc->desc.getElements().stream())// put all description element in one stream
 548                         .filter(element->element instanceof IndividualsAssociation)
 549                         .map(ia->(IndividualsAssociation)ia)
 550                         .collect(Collectors.toSet());
 551                 Classification classification = dataSet.getTaxonSubtreeFilter().iterator().next().getClassification();
 552                 for (IndividualsAssociation individualsAssociation : associations) {
 553                     if(individualsAssociation.getAssociatedSpecimenOrObservation().equals(descriptionBase.getDescribedSpecimenOrObservation())){
 554                         dataSet.removeDescription(individualsAssociation.getInDescription());
 555                         result.addDeletedObject(individualsAssociation.getInDescription());
 556                     }
 557                 }
 558             }
 559             result.addUpdatedObject(dataSet);
 560             result.setStatus(success?Status.OK:Status.ERROR);
 561         }
 562         return result;
 563     }
 564
 565     @Override
 566     @Transactional(readOnly=false)
 567     public TaxonRowWrapperDTO createTaxonDescription(UUID dataSetUuid, UUID taxonNodeUuid, DescriptionType descriptionType){
 568         DescriptiveDataSet dataSet = load(dataSetUuid);
 569         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid, Arrays.asList("taxon"));
 570         TaxonDescription newTaxonDescription = TaxonDescription.NewInstance(taxonNode.getTaxon());
 571         String tag = "";
 572         if(descriptionType.equals(DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION)){
 573             tag = "[Default]";
 574         }
 575         else if(descriptionType.equals(DescriptionType.SECONDARY_DATA)){
 576             tag = "[Literature]";
 577         }
 578         newTaxonDescription.setTitleCache(tag+" "+dataSet.getLabel()+": "+newTaxonDescription.generateTitle(), true); //$NON-NLS-2$
 579         newTaxonDescription.getTypes().add(descriptionType);
 580
 581         dataSet.getDescriptiveSystem().getDistinctTerms().forEach(wsFeature->{
 582             if(wsFeature.isSupportsCategoricalData()){
 583                 newTaxonDescription.addElement(CategoricalData.NewInstance(wsFeature));
 584             }
 585             else if(wsFeature.isSupportsQuantitativeData()){
 586                 newTaxonDescription.addElement(QuantitativeData.NewInstance(wsFeature));
 587             }
 588         });
 589         dataSet.addDescription(newTaxonDescription);
 590
 591         return createTaxonRowWrapper(newTaxonDescription.getUuid(), dataSet.getUuid());
 592     }
 593
 594     @Override
 595     public List<TermDto> getSupportedStatesForFeature(UUID featureUuid){
 596         return termDao.getSupportedStatesForFeature(featureUuid);
 597     }
 598
 599     @Override
 600     @Transactional(readOnly=false)
 601     public SpecimenDescription findSpecimenDescription(UUID descriptiveDataSetUuid, UUID specimenUuid, boolean addDatasetSource){
 602         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 603         SpecimenOrObservationBase specimen = occurrenceService.load(specimenUuid);
 604
 605         Set<? extends Feature> datasetFeatures = dataSet.getDescriptiveSystem().getDistinctTerms();
 606         List<DescriptionElementBase> matchingDescriptionElements = new ArrayList<>();
 607
 608         for (SpecimenDescription specimenDescription : (Set<SpecimenDescription>) specimen.getDescriptions()) {
 609             specimenDescription = (SpecimenDescription) descriptionService.load(specimenDescription.getUuid());
 610
 611             //check if description is already added to data set
 612             if(dataSet.getDescriptions().contains(specimenDescription)){
 613                 return specimenDescription;
 614             }
 615
 616             //gather specimen description features and check for match with dataset features
 617             Set<Feature> specimenDescriptionFeatures = new HashSet<>();
 618             for (DescriptionElementBase specimenDescriptionElement : specimenDescription.getElements()) {
 619                 Feature feature = specimenDescriptionElement.getFeature();
 620                 specimenDescriptionFeatures.add(feature);
 621                 if(datasetFeatures.contains(feature)){
 622                     matchingDescriptionElements.add(specimenDescriptionElement);
 623                 }
 624             }
 625         }
 626         //Create new specimen description if description has not already been added to the dataset
 627         SpecimenDescription newDesription = SpecimenDescription.NewInstance(specimen);
 628         newDesription.setTitleCache("Dataset "+dataSet.getLabel()+": "+newDesription.generateTitle(), true); //$NON-NLS-2$
 629
 630         //check for equals description element (same feature and same values)
 631         Map<Feature, List<DescriptionElementBase>> featureToElementMap = new HashMap<>();
 632         for(DescriptionElementBase element:matchingDescriptionElements){
 633             List<DescriptionElementBase> list = featureToElementMap.get(element.getFeature());
 634             if(list==null){
 635                 list = new ArrayList<>();
 636             }
 637             list.add(element);
 638             featureToElementMap.put(element.getFeature(), list);
 639         }
 640         Set<DescriptionElementBase> descriptionElementsToClone = new HashSet<>();
 641         for(Feature feature:featureToElementMap.keySet()){
 642             List<DescriptionElementBase> elements = featureToElementMap.get(feature);
 643             //no duplicate description elements found for this feature
 644             if(elements.size()==1){
 645                 descriptionElementsToClone.add(elements.get(0));
 646             }
 647             //duplicates found -> check if all are equal
 648             else{
 649                 DescriptionElementBase match = null;
 650                 for (DescriptionElementBase descriptionElementBase : elements) {
 651                     if(match==null){
 652                         match = descriptionElementBase;
 653                     }
 654                     else if(!new DescriptionElementCompareWrapper(match).equals(new DescriptionElementCompareWrapper(descriptionElementBase))){
 655                         match = null;
 656                         //TODO: propagate message
 657 //                        MessagingUtils.informationDialog(Messages.CharacterMatrix_MULTIPLE_DATA,
 658 //                                String.format(Messages.CharacterMatrix_MULTIPLE_DATA_MESSAGE, feature.getLabel()));
 659                         break;
 660                     }
 661                 }
 662                 if(match!=null){
 663                     descriptionElementsToClone.add(match);
 664                 }
 665             }
 666         }
 667         //clone matching descriptionElements
 668         for (DescriptionElementBase descriptionElementBase : descriptionElementsToClone) {
 669             DescriptionElementBase clone;
 670             try {
 671                 clone = descriptionElementBase.clone(newDesription);
 672                 clone.getSources().forEach(source -> {
 673                     if(descriptionElementBase instanceof CategoricalData){
 674                         TextData label = new DefaultCategoricalDescriptionBuilder().build((CategoricalData) descriptionElementBase, null);
 675                         source.setOriginalNameString(label.getText(Language.DEFAULT()));
 676                     }
 677                     else if(descriptionElementBase instanceof QuantitativeData){
 678                         TextData label = new DefaultQuantitativeDescriptionBuilder().build((QuantitativeData) descriptionElementBase, null);
 679                         source.setOriginalNameString(label.getText(Language.DEFAULT()));
 680                     }
 681                 });
 682             } catch (CloneNotSupportedException e) {
 683                 //nothing
 684             }
 685         }
 686
 687         //add all remaining description elements to the new description
 688         for(Feature wsFeature:datasetFeatures){
 689             boolean featureFound = false;
 690             for(DescriptionElementBase element:newDesription.getElements()){
 691                 if(element.getFeature().equals(wsFeature)){
 692                     featureFound = true;
 693                     break;
 694                 }
 695             }
 696             if(!featureFound){
 697                 if(wsFeature.isSupportsCategoricalData()){
 698                     newDesription.addElement(CategoricalData.NewInstance(wsFeature));
 699                 }
 700                 else if(wsFeature.isSupportsQuantitativeData()){
 701                     newDesription.addElement(QuantitativeData.NewInstance(wsFeature));
 702                 }
 703             }
 704         }
 705         //add sources of data set
 706         if(addDatasetSource){
 707             dataSet.getSources().forEach(source->{
 708                 try {
 709                     newDesription.addSource((IdentifiableSource) source.clone());
 710                 } catch (CloneNotSupportedException e) {
 711                     //nothing
 712                 }
 713             });
 714         }
 715         return newDesription;
 716
 717     }
 718
 719     //TODO: this should either be solved in the model class itself
 720     //OR this should cover all possibilities including modifiers for example
 721     private class DescriptionElementCompareWrapper {
 722
 723         private DescriptionElementBase element;
 724         private Set<UUID> stateUuids = new HashSet<>();
 725         private Set<Float> avgs = new HashSet<>();
 726         private Set<Float> exacts = new HashSet<>();
 727         private Set<Float> maxs = new HashSet<>();
 728         private Set<Float> mins = new HashSet<>();
 729         private Set<Float> sampleSizes = new HashSet<>();
 730         private Set<Float> standardDevs = new HashSet<>();
 731         private Set<Float> lowerBounds = new HashSet<>();
 732         private Set<Float> upperBounds = new HashSet<>();
 733         private Set<Float> variances = new HashSet<>();
 734
 735         public DescriptionElementCompareWrapper(DescriptionElementBase element) {
 736             this.element = element;
 737             if(element.isInstanceOf(CategoricalData.class)){
 738                 CategoricalData elementData = (CategoricalData)element;
 739                 elementData.getStatesOnly().forEach(state->stateUuids.add(state.getUuid()));
 740             }
 741             else if(element.isInstanceOf(QuantitativeData.class)){
 742                 QuantitativeData elementData = (QuantitativeData)element;
 743                 elementData.getStatisticalValues().forEach(value->{
 744                     if(value.getType().equals(StatisticalMeasure.AVERAGE())){
 745                         avgs.add(value.getValue());
 746                     }
 747                     else if(value.getType().equals(StatisticalMeasure.EXACT_VALUE())){
 748                         exacts.add(value.getValue());
 749
 750                     }
 751                     else if(value.getType().equals(StatisticalMeasure.MAX())){
 752                         maxs.add(value.getValue());
 753                     }
 754                     else if(value.getType().equals(StatisticalMeasure.MIN())){
 755                         mins.add(value.getValue());
 756                     }
 757                     else if(value.getType().equals(StatisticalMeasure.SAMPLE_SIZE())){
 758                         sampleSizes.add(value.getValue());
 759
 760                     }
 761                     else if(value.getType().equals(StatisticalMeasure.STANDARD_DEVIATION())){
 762                         standardDevs.add(value.getValue());
 763                     }
 764                     else if(value.getType().equals(StatisticalMeasure.TYPICAL_LOWER_BOUNDARY())){
 765                         lowerBounds.add(value.getValue());
 766
 767                     }
 768                     else if(value.getType().equals(StatisticalMeasure.TYPICAL_UPPER_BOUNDARY())){
 769                         upperBounds.add(value.getValue());
 770                     }
 771                     else if(value.getType().equals(StatisticalMeasure.VARIANCE())){
 772                         variances.add(value.getValue());
 773                     }
 774                 });
 775             }
 776         }
 777
 778         @Override
 779         public int hashCode() {
 780             final int prime = 31;
 781             int result = 1;
 782             result = prime * result + getOuterType().hashCode();
 783             result = prime * result + ((avgs == null) ? 0 : avgs.hashCode());
 784             result = prime * result + ((element == null) ? 0 : element.hashCode());
 785             result = prime * result + ((exacts == null) ? 0 : exacts.hashCode());
 786             result = prime * result + ((lowerBounds == null) ? 0 : lowerBounds.hashCode());
 787             result = prime * result + ((maxs == null) ? 0 : maxs.hashCode());
 788             result = prime * result + ((mins == null) ? 0 : mins.hashCode());
 789             result = prime * result + ((sampleSizes == null) ? 0 : sampleSizes.hashCode());
 790             result = prime * result + ((standardDevs == null) ? 0 : standardDevs.hashCode());
 791             result = prime * result + ((stateUuids == null) ? 0 : stateUuids.hashCode());
 792             result = prime * result + ((upperBounds == null) ? 0 : upperBounds.hashCode());
 793             result = prime * result + ((variances == null) ? 0 : variances.hashCode());
 794             return result;
 795         }
 796
 797         @Override
 798         public boolean equals(Object obj) {
 799             if (this == obj) {
 800                 return true;
 801             }
 802             if (obj == null) {
 803                 return false;
 804             }
 805             if (getClass() != obj.getClass()) {
 806                 return false;
 807             }
 808             DescriptionElementCompareWrapper other = (DescriptionElementCompareWrapper) obj;
 809             if (!getOuterType().equals(other.getOuterType())) {
 810                 return false;
 811             }
 812             if (avgs == null) {
 813                 if (other.avgs != null) {
 814                     return false;
 815                 }
 816             } else if (!avgs.equals(other.avgs)) {
 817                 return false;
 818             }
 819             if (element == null) {
 820                 if (other.element != null) {
 821                     return false;
 822                 }
 823             } else if (!element.equals(other.element)) {
 824                 return false;
 825             }
 826             if (exacts == null) {
 827                 if (other.exacts != null) {
 828                     return false;
 829                 }
 830             } else if (!exacts.equals(other.exacts)) {
 831                 return false;
 832             }
 833             if (lowerBounds == null) {
 834                 if (other.lowerBounds != null) {
 835                     return false;
 836                 }
 837             } else if (!lowerBounds.equals(other.lowerBounds)) {
 838                 return false;
 839             }
 840             if (maxs == null) {
 841                 if (other.maxs != null) {
 842                     return false;
 843                 }
 844             } else if (!maxs.equals(other.maxs)) {
 845                 return false;
 846             }
 847             if (mins == null) {
 848                 if (other.mins != null) {
 849                     return false;
 850                 }
 851             } else if (!mins.equals(other.mins)) {
 852                 return false;
 853             }
 854             if (sampleSizes == null) {
 855                 if (other.sampleSizes != null) {
 856                     return false;
 857                 }
 858             } else if (!sampleSizes.equals(other.sampleSizes)) {
 859                 return false;
 860             }
 861             if (standardDevs == null) {
 862                 if (other.standardDevs != null) {
 863                     return false;
 864                 }
 865             } else if (!standardDevs.equals(other.standardDevs)) {
 866                 return false;
 867             }
 868             if (stateUuids == null) {
 869                 if (other.stateUuids != null) {
 870                     return false;
 871                 }
 872             } else if (!stateUuids.equals(other.stateUuids)) {
 873                 return false;
 874             }
 875             if (upperBounds == null) {
 876                 if (other.upperBounds != null) {
 877                     return false;
 878                 }
 879             } else if (!upperBounds.equals(other.upperBounds)) {
 880                 return false;
 881             }
 882             if (variances == null) {
 883                 if (other.variances != null) {
 884                     return false;
 885                 }
 886             } else if (!variances.equals(other.variances)) {
 887                 return false;
 888             }
 889             return true;
 890         }
 891
 892         private DescriptiveDataSetService getOuterType() {
 893             return DescriptiveDataSetService.this;
 894         }
 895
 896     }
 897
 898 }