cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/DescriptiveDataSetService.java

   1 package eu.etaxonomy.cdm.api.service;
   2
   3 import java.util.ArrayList;
   4 import java.util.Arrays;
   5 import java.util.Collection;
   6 import java.util.HashMap;
   7 import java.util.HashSet;
   8 import java.util.List;
   9 import java.util.Map;
  10 import java.util.Map.Entry;
  11 import java.util.Optional;
  12 import java.util.Set;
  13 import java.util.UUID;
  14 import java.util.stream.Collectors;
  15
  16 import org.apache.log4j.Logger;
  17 import org.springframework.beans.factory.annotation.Autowired;
  18 import org.springframework.stereotype.Service;
  19 import org.springframework.transaction.annotation.Transactional;
  20
  21 import eu.etaxonomy.cdm.api.service.UpdateResult.Status;
  22 import eu.etaxonomy.cdm.api.service.config.DescriptionAggregationConfiguration;
  23 import eu.etaxonomy.cdm.api.service.config.IdentifiableServiceConfiguratorImpl;
  24 import eu.etaxonomy.cdm.api.service.dto.RowWrapperDTO;
  25 import eu.etaxonomy.cdm.api.service.dto.SpecimenRowWrapperDTO;
  26 import eu.etaxonomy.cdm.api.service.dto.TaxonRowWrapperDTO;
  27 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  28 import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
  29 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  30 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  31 import eu.etaxonomy.cdm.model.common.Language;
  32 import eu.etaxonomy.cdm.model.description.CategoricalData;
  33 import eu.etaxonomy.cdm.model.description.Character;
  34 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  35 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  36 import eu.etaxonomy.cdm.model.description.DescriptionType;
  37 import eu.etaxonomy.cdm.model.description.DescriptiveDataSet;
  38 import eu.etaxonomy.cdm.model.description.DescriptiveSystemRole;
  39 import eu.etaxonomy.cdm.model.description.Feature;
  40 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  41 import eu.etaxonomy.cdm.model.description.PolytomousKey;
  42 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  43 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  44 import eu.etaxonomy.cdm.model.description.StateData;
  45 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  46 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  47 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  48 import eu.etaxonomy.cdm.model.description.TextData;
  49 import eu.etaxonomy.cdm.model.location.NamedArea;
  50 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  51 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  52 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  53 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  54 import eu.etaxonomy.cdm.model.taxon.Classification;
  55 import eu.etaxonomy.cdm.model.taxon.Taxon;
  56 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  57 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  58 import eu.etaxonomy.cdm.persistence.dao.description.IDescriptiveDataSetDao;
  59 import eu.etaxonomy.cdm.persistence.dao.term.IDefinedTermDao;
  60 import eu.etaxonomy.cdm.persistence.dto.SpecimenNodeWrapper;
  61 import eu.etaxonomy.cdm.persistence.dto.TaxonNodeDto;
  62 import eu.etaxonomy.cdm.persistence.dto.TermDto;
  63 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  64 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
  65 import eu.etaxonomy.cdm.strategy.generate.PolytomousKeyGenerator;
  66 import eu.etaxonomy.cdm.strategy.generate.PolytomousKeyGeneratorConfigurator;
  67
  68 @Service
  69 @Transactional(readOnly=true)
  70 public class DescriptiveDataSetService
  71         extends IdentifiableServiceBase<DescriptiveDataSet, IDescriptiveDataSetDao>
  72         implements IDescriptiveDataSetService {
  73
  74     private static Logger logger = Logger.getLogger(DescriptiveDataSetService.class);
  75
  76     @Autowired
  77     private IOccurrenceService occurrenceService;
  78
  79     @Autowired
  80     private ITaxonService taxonService;
  81
  82     @Autowired
  83     private IPolytomousKeyService polytomousKeyService;
  84
  85     @Autowired
  86     private IDefinedTermDao termDao;
  87
  88     @Autowired
  89     private IDescriptionService descriptionService;
  90
  91     @Autowired
  92     private ITaxonNodeService taxonNodeService;
  93
  94         @Override
  95         @Autowired
  96         protected void setDao(IDescriptiveDataSetDao dao) {
  97                 this.dao = dao;
  98         }
  99
 100         @Override
 101     public Map<DescriptionBase, Set<DescriptionElementBase>> getDescriptionElements(DescriptiveDataSet descriptiveDataSet, Set<Feature> features, Integer pageSize,     Integer pageNumber,
 102                         List<String> propertyPaths) {
 103                 return dao.getDescriptionElements(descriptiveDataSet, features, pageSize, pageNumber, propertyPaths);
 104         }
 105
 106         @Override
 107         public <T extends DescriptionElementBase> Map<UuidAndTitleCache, Map<UUID, Set<T>>> getTaxonFeatureDescriptionElementMap(
 108                         Class<T> clazz, UUID descriptiveDataSetUuid, DescriptiveSystemRole role) {
 109                 return dao.getTaxonFeatureDescriptionElementMap(clazz, descriptiveDataSetUuid, role);
 110         }
 111
 112         @Override
 113     public List<UuidAndTitleCache<DescriptiveDataSet>> getDescriptiveDataSetUuidAndTitleCache(Integer limitOfInitialElements, String pattern) {
 114         return dao.getDescriptiveDataSetUuidAndTitleCache( limitOfInitialElements, pattern);
 115     }
 116
 117         @Override
 118         public ArrayList<RowWrapperDTO> getRowWrapper(UUID descriptiveDataSetUuid, IProgressMonitor monitor) {
 119             DescriptiveDataSet descriptiveDataSet = load(descriptiveDataSetUuid);
 120             monitor.beginTask("Load row wrapper", descriptiveDataSet.getDescriptions().size());
 121             ArrayList<RowWrapperDTO> wrappers = new ArrayList<>();
 122             Set<DescriptionBase> descriptions = descriptiveDataSet.getDescriptions();
 123             for (DescriptionBase description : descriptions) {
 124             if(monitor.isCanceled()){
 125                 return new ArrayList<>();
 126             }
 127             RowWrapperDTO rowWrapper = null;
 128             // only viable descriptions are aggregated, literature or default descriptions
 129             if(HibernateProxyHelper.isInstanceOf(description, TaxonDescription.class) &&
 130                     (description.getTypes().contains(DescriptionType.AGGREGATED)
 131                             || description.getTypes().contains(DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION)
 132                             || description.getTypes().contains(DescriptionType.SECONDARY_DATA)
 133                             )){
 134                 rowWrapper = createTaxonRowWrapper(description.getUuid(), descriptiveDataSet.getUuid());
 135             }
 136             else if (HibernateProxyHelper.isInstanceOf(description, SpecimenDescription.class)){
 137                 rowWrapper = createSpecimenRowWrapper(HibernateProxyHelper.deproxy(description, SpecimenDescription.class), descriptiveDataSetUuid);
 138             }
 139             if(rowWrapper!=null){
 140                 wrappers.add(rowWrapper);
 141             }
 142             monitor.worked(1);
 143         }
 144             return wrappers;
 145         }
 146
 147     @Override
 148     public Collection<SpecimenNodeWrapper> loadSpecimens(DescriptiveDataSet descriptiveDataSet){
 149         List<UUID> filteredNodes = findFilteredTaxonNodes(descriptiveDataSet);
 150         return occurrenceService.listUuidAndTitleCacheByAssociatedTaxon(filteredNodes, null, null);
 151     }
 152
 153     @Override
 154     public List<UUID> findFilteredTaxonNodes(DescriptiveDataSet descriptiveDataSet){
 155         TaxonNodeFilter filter = TaxonNodeFilter.NewRankInstance(descriptiveDataSet.getMinRank(), descriptiveDataSet.getMaxRank());
 156         descriptiveDataSet.getGeoFilter().forEach(area -> filter.orArea(area.getUuid()));
 157         descriptiveDataSet.getTaxonSubtreeFilter().forEach(node -> filter.orSubtree(node));
 158         filter.setIncludeUnpublished(true);
 159
 160         return taxonNodeService.uuidList(filter);
 161     }
 162
 163     @Override
 164     public List<TaxonNode> loadFilteredTaxonNodes(DescriptiveDataSet descriptiveDataSet, List<String> propertyPaths){
 165         return taxonNodeService.load(findFilteredTaxonNodes(descriptiveDataSet), propertyPaths);
 166     }
 167
 168     private TaxonNode findTaxonNodeForDescription(SpecimenDescription description, DescriptiveDataSet descriptiveDataSet){
 169         SpecimenOrObservationBase specimen = description.getDescribedSpecimenOrObservation();
 170         TaxonNode taxonNode = null;
 171         //get taxon node
 172
 173         Set<IndividualsAssociation> associations = (Set<IndividualsAssociation>) descriptiveDataSet.getDescriptions()
 174                 .stream()
 175                 .flatMap(desc->desc.getElements().stream())// put all description element in one stream
 176                 .filter(element->element instanceof IndividualsAssociation)
 177                 .map(ia->(IndividualsAssociation)ia)
 178                 .collect(Collectors.toSet());
 179         Classification classification = descriptiveDataSet.getTaxonSubtreeFilter().iterator().next().getClassification();
 180         for (IndividualsAssociation individualsAssociation : associations) {
 181             if(individualsAssociation.getAssociatedSpecimenOrObservation().equals(specimen)){
 182                 return ((TaxonDescription) individualsAssociation.getInDescription()).getTaxon().getTaxonNode(classification);
 183             }
 184         }
 185         return null;
 186     }
 187
 188     private TaxonNode findTaxonNodeForSpecimen(TaxonNode taxonNode, SpecimenOrObservationBase<?> specimen){
 189         Collection<SpecimenNodeWrapper> nodeWrapper = occurrenceService.listUuidAndTitleCacheByAssociatedTaxon(Arrays.asList(taxonNode.getUuid()), null, null);
 190         for (SpecimenNodeWrapper specimenNodeWrapper : nodeWrapper) {
 191             if(specimenNodeWrapper.getUuidAndTitleCache().getId().equals(specimen.getId())){
 192                 return taxonNode;
 193             }
 194         }
 195         return null;
 196     }
 197
 198     @Override
 199     public TaxonRowWrapperDTO createTaxonRowWrapper(UUID taxonDescriptionUuid, UUID descriptiveDataSetUuid) {
 200         TaxonNode taxonNode = null;
 201         Classification classification = null;
 202         TaxonDescription description = (TaxonDescription) descriptionService.load(taxonDescriptionUuid,
 203                 Arrays.asList("taxon", "descriptionElements", "descriptionElements.feature"));
 204         DescriptiveDataSet descriptiveDataSet = dao.load(descriptiveDataSetUuid, null);
 205         Optional<TaxonNode> first = descriptiveDataSet.getTaxonSubtreeFilter().stream()
 206                 .filter(node->node.getClassification()!=null).findFirst();
 207         Optional<Classification> classificationOptional = first.map(node->node.getClassification());
 208         if(classificationOptional.isPresent()){
 209             classification = classificationOptional.get();
 210             Taxon taxon = (Taxon) taxonService.load(description.getTaxon().getId(), Arrays.asList("taxonNodes", "taxonNodes.classification"));
 211             taxonNode = taxon.getTaxonNode(classification);
 212         }
 213         return new TaxonRowWrapperDTO(description, new TaxonNodeDto(taxonNode));
 214     }
 215
 216     @Override
 217     @Transactional(readOnly=false)
 218     public UpdateResult addRowWrapperToDataset(Collection<SpecimenNodeWrapper> wrappers, UUID datasetUuid){
 219         UpdateResult result = new UpdateResult();
 220         DescriptiveDataSet dataSet = load(datasetUuid);
 221         result.setCdmEntity(dataSet);
 222         for (SpecimenNodeWrapper wrapper : wrappers) {
 223             UUID taxonDescriptionUuid = wrapper.getTaxonDescriptionUuid();
 224             TaxonDescription taxonDescription = null;
 225             if(taxonDescriptionUuid!=null){
 226                 taxonDescription = (TaxonDescription) descriptionService.load(taxonDescriptionUuid);
 227             }
 228             if(taxonDescription==null){
 229                 Optional<TaxonDescription> associationDescriptionOptional = wrapper.getTaxonNode().getTaxon().getDescriptions().stream()
 230                         .filter(desc->desc.getTypes().contains(DescriptionType.INDIVIDUALS_ASSOCIATION))
 231                         .findFirst();
 232                 Taxon taxon = wrapper.getTaxonNode().getTaxon();
 233                 if(!associationDescriptionOptional.isPresent()){
 234                     taxonDescription = TaxonDescription.NewInstance(taxon);
 235                 }
 236                 else{
 237                     taxonDescription = associationDescriptionOptional.get();
 238                 }
 239
 240                 SpecimenOrObservationBase specimen = occurrenceService.load(wrapper.getUuidAndTitleCache().getUuid());
 241                 IndividualsAssociation association = IndividualsAssociation.NewInstance(specimen);
 242                 taxonDescription.addElement(association);
 243                 taxonService.saveOrUpdate(taxon);
 244                 result.addUpdatedObject(taxon);
 245             }
 246             SpecimenDescription specimenDescription = findSpecimenDescription(datasetUuid, wrapper.getUuidAndTitleCache().getUuid(), true);
 247             SpecimenRowWrapperDTO rowWrapper = createSpecimenRowWrapper(specimenDescription, wrapper.getTaxonNode().getUuid(), datasetUuid);
 248             if(rowWrapper==null){
 249                 result.addException(new IllegalArgumentException("Could not create wrapper for "+specimenDescription));
 250                 continue;
 251             }
 252             //add specimen description to data set
 253             rowWrapper.getDescription().addDescriptiveDataSet(dataSet);
 254             //add taxon description with IndividualsAssociation to the specimen to data set
 255             taxonDescription.addDescriptiveDataSet(dataSet);
 256
 257             result.addUpdatedObject(rowWrapper.getDescription());
 258             result.addUpdatedObject(taxonDescription);
 259         }
 260         saveOrUpdate(dataSet);
 261         return result;
 262     }
 263
 264     private SpecimenRowWrapperDTO createSpecimenRowWrapper(SpecimenDescription description, UUID taxonNodeUuid,
 265             UUID datasetUuid) {
 266         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid);
 267         DescriptiveDataSet descriptiveDataSet = load(datasetUuid);
 268         SpecimenOrObservationBase specimen = description.getDescribedSpecimenOrObservation();
 269         //supplemental information
 270         if(taxonNode==null){
 271             taxonNode = findTaxonNodeForDescription(description, descriptiveDataSet);
 272         }
 273         FieldUnit fieldUnit = null;
 274         String identifier = null;
 275         NamedArea country = null;
 276         if(taxonNode==null){
 277             return null;
 278         }
 279         //taxon node was found
 280
 281         //get field unit
 282         Collection<FieldUnit> fieldUnits = occurrenceService.findFieldUnits(specimen.getUuid(),
 283                 Arrays.asList(new String[]{
 284                         "gatheringEvent",
 285                         "gatheringEvent.country"
 286                 }));
 287         if(fieldUnits.size()!=1){
 288             logger.error("More than one or no field unit found for specimen"); //$NON-NLS-1$
 289             return null;
 290         }
 291         else{
 292             fieldUnit = fieldUnits.iterator().next();
 293         }
 294         //get identifier
 295         if(HibernateProxyHelper.isInstanceOf(specimen, DerivedUnit.class)){
 296             identifier = occurrenceService.getMostSignificantIdentifier(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class));
 297         }
 298         //get country
 299         if(fieldUnit!=null && fieldUnit.getGatheringEvent()!=null){
 300             country = fieldUnit.getGatheringEvent().getCountry();
 301         }
 302         //get default taxon description
 303         TaxonDescription defaultTaxonDescription = findTaxonDescriptionByDescriptionType(descriptiveDataSet.getUuid(),
 304                 taxonNode.getUuid(), DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION);
 305         TaxonRowWrapperDTO taxonRowWrapper = defaultTaxonDescription != null
 306                 ? createTaxonRowWrapper(defaultTaxonDescription.getUuid(), descriptiveDataSet.getUuid()) : null;
 307         return new SpecimenRowWrapperDTO(description, new TaxonNodeDto(taxonNode), fieldUnit, identifier, country);
 308     }
 309
 310     @Override
 311     public SpecimenRowWrapperDTO createSpecimenRowWrapper(SpecimenDescription description, UUID descriptiveDataSetUuid){
 312         return createSpecimenRowWrapper(description, null, descriptiveDataSetUuid);
 313         }
 314
 315     @Override
 316     @Transactional(readOnly = false)
 317     public UpdateResult updateCaches(Class<? extends DescriptiveDataSet> clazz, Integer stepSize,
 318             IIdentifiableEntityCacheStrategy<DescriptiveDataSet> cacheStrategy, IProgressMonitor monitor) {
 319         if (clazz == null) {
 320             clazz = DescriptiveDataSet.class;
 321         }
 322         return super.updateCachesImpl(clazz, stepSize, cacheStrategy, monitor);
 323     }
 324
 325     private TaxonDescription findTaxonDescriptionByDescriptionType(DescriptiveDataSet dataSet, Taxon taxon, DescriptionType descriptionType){
 326         Set<DescriptionBase> dataSetDescriptions = dataSet.getDescriptions();
 327         //filter by DEFAULT descriptions
 328         Optional<TaxonDescription> first = taxon.getDescriptions().stream()
 329                 .filter(desc -> desc.getTypes().stream().anyMatch(type -> type.equals(descriptionType)))
 330                 .filter(defaultDescription->dataSetDescriptions.contains(defaultDescription))
 331                 .findFirst();
 332         if(first.isPresent()){
 333             return HibernateProxyHelper.deproxy(descriptionService.load(first.get().getUuid(),
 334                   Arrays.asList("taxon", "descriptionElements", "descriptionElements.feature")), TaxonDescription.class);
 335         }
 336         return null;
 337     }
 338
 339     @Override
 340     public TaxonDescription findTaxonDescriptionByDescriptionType(UUID dataSetUuid, UUID taxonNodeUuid, DescriptionType descriptionType){
 341         DescriptiveDataSet dataSet = load(dataSetUuid);
 342         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid);
 343         return findTaxonDescriptionByDescriptionType(dataSet, taxonNode.getTaxon(), descriptionType);
 344     }
 345
 346     @Override
 347     @Transactional(readOnly=false)
 348     public UpdateResult aggregate(UUID descriptiveDataSetUuid, DescriptionAggregationConfiguration config, IProgressMonitor monitor) {
 349         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 350         Set<DescriptionBase> descriptions = dataSet.getDescriptions();
 351
 352         monitor.beginTask("Aggregate data set", descriptions.size()*2);
 353
 354         UpdateResult result = new UpdateResult();
 355         result.setCdmEntity(dataSet);
 356
 357         // delete all aggregation description of this dataset (DescriptionType.AGGREGATED)
 358         Set<DescriptionBase> aggregations = dataSet.getDescriptions().stream()
 359         .filter(aggDesc->aggDesc instanceof TaxonDescription)
 360         .filter(desc -> desc.getTypes().stream().anyMatch(type -> type.equals(DescriptionType.AGGREGATED)))
 361         .collect(Collectors.toSet());
 362
 363         aggregations.forEach(aggregation->dataSet.removeDescription(aggregation));
 364
 365         // clone specimen descriptions
 366         // create a snapshot of those descriptions that were used to create the aggregated descriptions
 367         // TODO implement when the clones descriptions can be attached to taxon descriptions as sources
 368 //        for (DescriptionBase<?> descriptionBase : descriptions) {
 369 //            if(descriptionBase instanceof SpecimenDescription){
 370 //                SpecimenDescription specimenDescription = (SpecimenDescription)descriptionBase;
 371 //                SpecimenOrObservationBase<?> specimenOrObservation = specimenDescription.getDescribedSpecimenOrObservation();
 372 //                SpecimenDescription clone = (SpecimenDescription) specimenDescription.clone();
 373 //                clone.setDescriptionType(DescriptionType.AggregationClone);
 374 //                specimenOrObservation.addDescription(clone);
 375 //            }
 376 //        }
 377
 378         // sort descriptions by taxa
 379         Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap = new HashMap<>();
 380         for (DescriptionBase descriptionBase : descriptions) {
 381             if(monitor.isCanceled()){
 382                 result.setAbort();
 383                 return result;
 384             }
 385
 386             if(descriptionBase instanceof SpecimenDescription){
 387                 SpecimenDescription specimenDescription = HibernateProxyHelper.deproxy(descriptionBase, SpecimenDescription.class);
 388                 if(specimenDescription.getElements().stream().anyMatch(element->hasCharacterData(element))){
 389                     TaxonNode taxonNode = findTaxonNodeForDescription(specimenDescription, dataSet);
 390                     if(taxonNode!=null){
 391                         addDescriptionToTaxonNodeMap(specimenDescription.getUuid(), taxonNode, taxonNodeToSpecimenDescriptionMap);
 392                     }
 393                 }
 394             }
 395             monitor.worked(1);
 396         }
 397         if(config.isRecursiveAggregation()){
 398             propagateDescriptionsToParentNodes(dataSet, taxonNodeToSpecimenDescriptionMap);
 399         }
 400         // aggregate per taxa
 401         for (Entry<TaxonNode, Set<UUID>> entry: taxonNodeToSpecimenDescriptionMap.entrySet()) {
 402             if(monitor.isCanceled()){
 403                 result.setAbort();
 404                 return result;
 405             }
 406             UUID taxonUuid = entry.getKey().getTaxon().getUuid();
 407             Set<UUID> specimenDescriptionUuids = entry.getValue();
 408             result.includeResult(aggregateDescription(taxonUuid , specimenDescriptionUuids, descriptiveDataSetUuid));
 409             monitor.worked(1);
 410         }
 411         monitor.done();
 412         return result;
 413     }
 414
 415
 416     private boolean hasCharacterData(DescriptionElementBase element) {
 417         return (element instanceof CategoricalData && !((CategoricalData) element).getStatesOnly().isEmpty())
 418                 || (element instanceof QuantitativeData
 419                         && !((QuantitativeData) element).getStatisticalValues().isEmpty());
 420     }
 421
 422     private void addDescriptionToTaxonNodeMap(UUID descriptionUuid, TaxonNode taxonNode, Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap){
 423         Set<UUID> specimenDescriptionUuids = taxonNodeToSpecimenDescriptionMap.get(taxonNode);
 424         if(specimenDescriptionUuids==null){
 425             specimenDescriptionUuids = new HashSet<>();
 426         }
 427         specimenDescriptionUuids.add(descriptionUuid);
 428         taxonNodeToSpecimenDescriptionMap.put(taxonNode, specimenDescriptionUuids);
 429     }
 430
 431     private void propagateDescriptionsToParentNodes(DescriptiveDataSet dataSet, Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap){
 432         Map<TaxonNode, Set<UUID>> parentMap = new HashMap<>();
 433         for (Entry<TaxonNode, Set<UUID>> entry: taxonNodeToSpecimenDescriptionMap.entrySet()) {
 434             Set<UUID> descriptionUuids = entry.getValue();
 435             TaxonNode node = entry.getKey();
 436             TaxonNode parentNode = node.getParent();
 437             while(parentNode!=null && isTaxonNodeInDescriptiveDataSet(parentNode, dataSet)){
 438                 for (UUID uuid : descriptionUuids) {
 439                     addDescriptionToTaxonNodeMap(uuid, node.getParent(), parentMap);
 440                 }
 441                 parentNode = parentNode.getParent();
 442             }
 443         }
 444         // merge parent map
 445         for (Entry<TaxonNode, Set<UUID>> entry: parentMap.entrySet()) {
 446             Set<UUID> descriptionUuids = entry.getValue();
 447             TaxonNode node = entry.getKey();
 448             for (UUID uuid : descriptionUuids) {
 449                 addDescriptionToTaxonNodeMap(uuid, node, taxonNodeToSpecimenDescriptionMap);
 450             }
 451         }
 452     }
 453
 454     private boolean isTaxonNodeInDescriptiveDataSet(TaxonNode taxonNode, DescriptiveDataSet dataSet){
 455         Set<TaxonNode> taxonSubtreeFilter = dataSet.getTaxonSubtreeFilter();
 456         for (TaxonNode datasetNode : taxonSubtreeFilter) {
 457             if(datasetNode.getUuid().equals(taxonNode.getUuid())){
 458                 return true;
 459             }
 460             List<TaxonNode> allChildren = taxonNodeService.loadChildNodesOfTaxonNode(datasetNode, null, true, true, null);
 461             for (TaxonNode childNode : allChildren) {
 462                 if(childNode.getUuid().equals(taxonNode.getUuid())){
 463                     return true;
 464                 }
 465             }
 466         }
 467         return false;
 468     }
 469
 470     @Override
 471     @Transactional(readOnly=false)
 472     public UpdateResult generatePolytomousKey(UUID descriptiveDataSetUuid, UUID taxonUuid) {
 473         UpdateResult result = new UpdateResult();
 474
 475         PolytomousKeyGeneratorConfigurator keyConfig = new PolytomousKeyGeneratorConfigurator();
 476         DescriptiveDataSet descriptiveDataSet = load(descriptiveDataSetUuid);
 477         keyConfig.setDataSet(descriptiveDataSet);
 478         PolytomousKey key = new PolytomousKeyGenerator().invoke(keyConfig);
 479         IdentifiableServiceConfiguratorImpl<PolytomousKey> serviceConfig= new IdentifiableServiceConfiguratorImpl<>();
 480         serviceConfig.setTitleSearchString(descriptiveDataSet.getTitleCache());
 481         List<PolytomousKey> list = polytomousKeyService.findByTitle(serviceConfig).getRecords();
 482         if(list!=null){
 483             list.forEach(polytomousKey->polytomousKeyService.delete(polytomousKey));
 484         }
 485         key.setTitleCache(descriptiveDataSet.getTitleCache(), true);
 486
 487         Taxon taxon = (Taxon) taxonService.load(taxonUuid);
 488         key.addTaxonomicScope(taxon);
 489
 490         polytomousKeyService.saveOrUpdate(key);
 491
 492         result.setCdmEntity(key);
 493         result.addUpdatedObject(taxon);
 494         return result;
 495     }
 496
 497     @SuppressWarnings("unchecked")
 498     private UpdateResult aggregateDescription(UUID taxonUuid, Set<UUID> descriptionUuids, UUID descriptiveDataSetUuid) {
 499         UpdateResult result = new UpdateResult();
 500
 501         TaxonBase taxonBase = taxonService.load(taxonUuid);
 502         if(!(taxonBase instanceof Taxon)){
 503             result.addException(new ClassCastException("The given taxonUUID does not belong to a taxon"));
 504             result.setError();
 505             return result;
 506         }
 507         Taxon taxon = (Taxon)taxonBase;
 508         List<DescriptionBase> descriptions = descriptionService.load(new ArrayList<>(descriptionUuids), null);
 509         Map<Character, List<DescriptionElementBase>> featureToElementMap = new HashMap<>();
 510
 511         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 512         if(dataSet==null){
 513             result.addException(new IllegalArgumentException("Could not find data set for uuid "+descriptiveDataSetUuid));
 514             result.setAbort();
 515             return result;
 516         }
 517
 518         //extract all character description elements
 519         descriptions.forEach(description->{
 520             description.getElements()
 521             .stream()
 522             //filter out elements that do not have a Character as Feature
 523             .filter(element->HibernateProxyHelper.isInstanceOf(((DescriptionElementBase)element).getFeature(), Character.class))
 524             .forEach(ele->{
 525                 DescriptionElementBase descriptionElement = (DescriptionElementBase)ele;
 526                 List<DescriptionElementBase> list = featureToElementMap.get(descriptionElement.getFeature());
 527                 if(list==null){
 528                     list = new ArrayList<>();
 529                 }
 530                 list.add(descriptionElement);
 531                 featureToElementMap.put(HibernateProxyHelper.deproxy(descriptionElement.getFeature(), Character.class), list);
 532             });
 533         });
 534
 535         // create new aggregation
 536         TaxonDescription description = TaxonDescription.NewInstance(taxon);
 537         description.setTitleCache("[Aggregation] "+dataSet.getTitleCache(), true);
 538         description.getTypes().add(DescriptionType.AGGREGATED);
 539         IdentifiableSource source = IdentifiableSource.NewInstance(OriginalSourceType.Aggregation);
 540         description.addSource(source);
 541         description.addDescriptiveDataSet(dataSet);
 542
 543         featureToElementMap.forEach((feature, elements)->{
 544             //aggregate categorical data
 545             if(feature.isSupportsCategoricalData()){
 546                 CategoricalData aggregate = CategoricalData.NewInstance(feature);
 547                 elements.stream()
 548                 .filter(element->element instanceof CategoricalData)
 549                 .forEach(categoricalData->((CategoricalData)categoricalData).getStateData()
 550                         .forEach(stateData->aggregate.addStateData((StateData) stateData.clone())));
 551                 description.addElement(aggregate);
 552             }
 553             //aggregate quantitative data
 554             else if(feature.isSupportsQuantitativeData()){
 555                 QuantitativeData aggregate = QuantitativeData.NewInstance(feature);
 556                 elements.stream()
 557                 .filter(element->element instanceof QuantitativeData)
 558                 .forEach(categoricalData->((QuantitativeData)categoricalData).getStatisticalValues()
 559                         .forEach(statisticalValue->aggregate.addStatisticalValue((StatisticalMeasurementValue) statisticalValue.clone())));
 560                 description.addElement(aggregate);
 561             }
 562         });
 563         result.addUpdatedObject(taxon);
 564         result.addUpdatedObject(description);
 565
 566         return result;
 567     }
 568
 569     @Override
 570     @Transactional(readOnly=false)
 571     public DeleteResult removeDescription(UUID descriptionUuid, UUID descriptiveDataSetUuid) {
 572         DeleteResult result = new DeleteResult();
 573         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 574         DescriptionBase descriptionBase = descriptionService.load(descriptionUuid);
 575         if(dataSet==null || descriptionBase==null){
 576             result.setError();
 577         }
 578         else{
 579             boolean success = dataSet.removeDescription(descriptionBase);
 580             result.addDeletedObject(descriptionBase);
 581             // remove taxon description with IndividualsAssociation from data set
 582             if(descriptionBase instanceof SpecimenDescription){
 583                 Set<IndividualsAssociation> associations = (Set<IndividualsAssociation>) dataSet.getDescriptions()
 584                         .stream()
 585                         .flatMap(desc->desc.getElements().stream())// put all description element in one stream
 586                         .filter(element->element instanceof IndividualsAssociation)
 587                         .map(ia->(IndividualsAssociation)ia)
 588                         .collect(Collectors.toSet());
 589                 Classification classification = dataSet.getTaxonSubtreeFilter().iterator().next().getClassification();
 590                 for (IndividualsAssociation individualsAssociation : associations) {
 591                     if(individualsAssociation.getAssociatedSpecimenOrObservation().equals(descriptionBase.getDescribedSpecimenOrObservation())){
 592                         dataSet.removeDescription(individualsAssociation.getInDescription());
 593                         result.addDeletedObject(individualsAssociation.getInDescription());
 594                     }
 595                 }
 596             }
 597             result.addUpdatedObject(dataSet);
 598             result.setStatus(success?Status.OK:Status.ERROR);
 599         }
 600         return result;
 601     }
 602
 603     @Override
 604     @Transactional(readOnly=false)
 605     public TaxonRowWrapperDTO createTaxonDescription(UUID dataSetUuid, UUID taxonNodeUuid, DescriptionType descriptionType){
 606         DescriptiveDataSet dataSet = load(dataSetUuid);
 607         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid, Arrays.asList("taxon"));
 608         TaxonDescription newTaxonDescription = TaxonDescription.NewInstance(taxonNode.getTaxon());
 609         String tag = "";
 610         if(descriptionType.equals(DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION)){
 611             tag = "[Default]";
 612         }
 613         else if(descriptionType.equals(DescriptionType.SECONDARY_DATA)){
 614             tag = "[Literature]";
 615         }
 616         newTaxonDescription.setTitleCache(tag+" "+dataSet.getLabel()+": "+newTaxonDescription.generateTitle(), true); //$NON-NLS-2$
 617         newTaxonDescription.getTypes().add(descriptionType);
 618
 619         dataSet.getDescriptiveSystem().getDistinctTerms().forEach(wsFeature->{
 620             if(wsFeature.isSupportsCategoricalData()){
 621                 newTaxonDescription.addElement(CategoricalData.NewInstance(wsFeature));
 622             }
 623             else if(wsFeature.isSupportsQuantitativeData()){
 624                 newTaxonDescription.addElement(QuantitativeData.NewInstance(wsFeature));
 625             }
 626         });
 627         dataSet.addDescription(newTaxonDescription);
 628
 629         return createTaxonRowWrapper(newTaxonDescription.getUuid(), dataSet.getUuid());
 630     }
 631
 632     @Override
 633     public List<TermDto> getSupportedStatesForFeature(UUID featureUuid){
 634         return termDao.getSupportedStatesForFeature(featureUuid);
 635     }
 636
 637     @Override
 638     @Transactional(readOnly=false)
 639     public SpecimenDescription findSpecimenDescription(UUID descriptiveDataSetUuid, UUID specimenUuid, boolean addDatasetSource){
 640         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 641         SpecimenOrObservationBase specimen = occurrenceService.load(specimenUuid);
 642
 643         Set<? extends Feature> datasetFeatures = dataSet.getDescriptiveSystem().getDistinctTerms();
 644         List<DescriptionElementBase> matchingDescriptionElements = new ArrayList<>();
 645
 646         for (SpecimenDescription specimenDescription : (Set<SpecimenDescription>) specimen.getDescriptions()) {
 647             specimenDescription = (SpecimenDescription) descriptionService.load(specimenDescription.getUuid());
 648
 649             //check if description is already added to data set
 650             if(dataSet.getDescriptions().contains(specimenDescription)){
 651                 return specimenDescription;
 652             }
 653
 654             //gather specimen description features and check for match with dataset features
 655             Set<Feature> specimenDescriptionFeatures = new HashSet<>();
 656             for (DescriptionElementBase specimenDescriptionElement : specimenDescription.getElements()) {
 657                 Feature feature = specimenDescriptionElement.getFeature();
 658                 specimenDescriptionFeatures.add(feature);
 659                 if(datasetFeatures.contains(feature)){
 660                     matchingDescriptionElements.add(specimenDescriptionElement);
 661                 }
 662             }
 663         }
 664         //Create new specimen description if description has not already been added to the dataset
 665         SpecimenDescription newDesription = SpecimenDescription.NewInstance(specimen);
 666         newDesription.setTitleCache("Dataset "+dataSet.getLabel()+": "+newDesription.generateTitle(), true); //$NON-NLS-2$
 667
 668         //check for equals description element (same feature and same values)
 669         Map<Feature, List<DescriptionElementBase>> featureToElementMap = new HashMap<>();
 670         for(DescriptionElementBase element:matchingDescriptionElements){
 671             List<DescriptionElementBase> list = featureToElementMap.get(element.getFeature());
 672             if(list==null){
 673                 list = new ArrayList<>();
 674             }
 675             list.add(element);
 676             featureToElementMap.put(element.getFeature(), list);
 677         }
 678         Set<DescriptionElementBase> descriptionElementsToClone = new HashSet<>();
 679         for(Feature feature:featureToElementMap.keySet()){
 680             List<DescriptionElementBase> elements = featureToElementMap.get(feature);
 681             //no duplicate description elements found for this feature
 682             if(elements.size()==1){
 683                 descriptionElementsToClone.add(elements.get(0));
 684             }
 685             //duplicates found -> check if all are equal
 686             else{
 687                 DescriptionElementBase match = null;
 688                 for (DescriptionElementBase descriptionElementBase : elements) {
 689                     if(match==null){
 690                         match = descriptionElementBase;
 691                     }
 692                     else if(!new DescriptionElementCompareWrapper(match).equals(new DescriptionElementCompareWrapper(descriptionElementBase))){
 693                         match = null;
 694                         //TODO: propagate message
 695 //                        MessagingUtils.informationDialog(Messages.CharacterMatrix_MULTIPLE_DATA,
 696 //                                String.format(Messages.CharacterMatrix_MULTIPLE_DATA_MESSAGE, feature.getLabel()));
 697                         break;
 698                     }
 699                 }
 700                 if(match!=null){
 701                     descriptionElementsToClone.add(match);
 702                 }
 703             }
 704         }
 705         //clone matching descriptionElements
 706         for (DescriptionElementBase descriptionElementBase : descriptionElementsToClone) {
 707             DescriptionElementBase clone;
 708             try {
 709                 clone = descriptionElementBase.clone(newDesription);
 710                 clone.getSources().forEach(source -> {
 711                     if(descriptionElementBase instanceof CategoricalData){
 712                         TextData label = new DefaultCategoricalDescriptionBuilder().build((CategoricalData) descriptionElementBase, null);
 713                         source.setOriginalNameString(label.getText(Language.DEFAULT()));
 714                     }
 715                     else if(descriptionElementBase instanceof QuantitativeData){
 716                         TextData label = new DefaultQuantitativeDescriptionBuilder().build((QuantitativeData) descriptionElementBase, null);
 717                         source.setOriginalNameString(label.getText(Language.DEFAULT()));
 718                     }
 719                 });
 720             } catch (CloneNotSupportedException e) {
 721                 //nothing
 722             }
 723         }
 724
 725         //add all remaining description elements to the new description
 726         for(Feature wsFeature:datasetFeatures){
 727             boolean featureFound = false;
 728             for(DescriptionElementBase element:newDesription.getElements()){
 729                 if(element.getFeature().equals(wsFeature)){
 730                     featureFound = true;
 731                     break;
 732                 }
 733             }
 734             if(!featureFound){
 735                 if(wsFeature.isSupportsCategoricalData()){
 736                     newDesription.addElement(CategoricalData.NewInstance(wsFeature));
 737                 }
 738                 else if(wsFeature.isSupportsQuantitativeData()){
 739                     newDesription.addElement(QuantitativeData.NewInstance(wsFeature));
 740                 }
 741             }
 742         }
 743         //add sources of data set
 744         if(addDatasetSource){
 745             dataSet.getSources().forEach(source->{
 746                 try {
 747                     newDesription.addSource((IdentifiableSource) source.clone());
 748                 } catch (CloneNotSupportedException e) {
 749                     //nothing
 750                 }
 751             });
 752         }
 753         return newDesription;
 754
 755     }
 756
 757     //TODO: this should either be solved in the model class itself
 758     //OR this should cover all possibilities including modifiers for example
 759     private class DescriptionElementCompareWrapper {
 760
 761         private DescriptionElementBase element;
 762         private Set<UUID> stateUuids = new HashSet<>();
 763         private Set<Float> avgs = new HashSet<>();
 764         private Set<Float> exacts = new HashSet<>();
 765         private Set<Float> maxs = new HashSet<>();
 766         private Set<Float> mins = new HashSet<>();
 767         private Set<Float> sampleSizes = new HashSet<>();
 768         private Set<Float> standardDevs = new HashSet<>();
 769         private Set<Float> lowerBounds = new HashSet<>();
 770         private Set<Float> upperBounds = new HashSet<>();
 771         private Set<Float> variances = new HashSet<>();
 772
 773         public DescriptionElementCompareWrapper(DescriptionElementBase element) {
 774             this.element = element;
 775             if(element.isInstanceOf(CategoricalData.class)){
 776                 CategoricalData elementData = (CategoricalData)element;
 777                 elementData.getStatesOnly().forEach(state->stateUuids.add(state.getUuid()));
 778             }
 779             else if(element.isInstanceOf(QuantitativeData.class)){
 780                 QuantitativeData elementData = (QuantitativeData)element;
 781                 elementData.getStatisticalValues().forEach(value->{
 782                     if(value.getType().equals(StatisticalMeasure.AVERAGE())){
 783                         avgs.add(value.getValue());
 784                     }
 785                     else if(value.getType().equals(StatisticalMeasure.EXACT_VALUE())){
 786                         exacts.add(value.getValue());
 787
 788                     }
 789                     else if(value.getType().equals(StatisticalMeasure.MAX())){
 790                         maxs.add(value.getValue());
 791                     }
 792                     else if(value.getType().equals(StatisticalMeasure.MIN())){
 793                         mins.add(value.getValue());
 794                     }
 795                     else if(value.getType().equals(StatisticalMeasure.SAMPLE_SIZE())){
 796                         sampleSizes.add(value.getValue());
 797
 798                     }
 799                     else if(value.getType().equals(StatisticalMeasure.STANDARD_DEVIATION())){
 800                         standardDevs.add(value.getValue());
 801                     }
 802                     else if(value.getType().equals(StatisticalMeasure.TYPICAL_LOWER_BOUNDARY())){
 803                         lowerBounds.add(value.getValue());
 804
 805                     }
 806                     else if(value.getType().equals(StatisticalMeasure.TYPICAL_UPPER_BOUNDARY())){
 807                         upperBounds.add(value.getValue());
 808                     }
 809                     else if(value.getType().equals(StatisticalMeasure.VARIANCE())){
 810                         variances.add(value.getValue());
 811                     }
 812                 });
 813             }
 814         }
 815
 816         @Override
 817         public int hashCode() {
 818             final int prime = 31;
 819             int result = 1;
 820             result = prime * result + getOuterType().hashCode();
 821             result = prime * result + ((avgs == null) ? 0 : avgs.hashCode());
 822             result = prime * result + ((element == null) ? 0 : element.hashCode());
 823             result = prime * result + ((exacts == null) ? 0 : exacts.hashCode());
 824             result = prime * result + ((lowerBounds == null) ? 0 : lowerBounds.hashCode());
 825             result = prime * result + ((maxs == null) ? 0 : maxs.hashCode());
 826             result = prime * result + ((mins == null) ? 0 : mins.hashCode());
 827             result = prime * result + ((sampleSizes == null) ? 0 : sampleSizes.hashCode());
 828             result = prime * result + ((standardDevs == null) ? 0 : standardDevs.hashCode());
 829             result = prime * result + ((stateUuids == null) ? 0 : stateUuids.hashCode());
 830             result = prime * result + ((upperBounds == null) ? 0 : upperBounds.hashCode());
 831             result = prime * result + ((variances == null) ? 0 : variances.hashCode());
 832             return result;
 833         }
 834
 835         @Override
 836         public boolean equals(Object obj) {
 837             if (this == obj) {
 838                 return true;
 839             }
 840             if (obj == null) {
 841                 return false;
 842             }
 843             if (getClass() != obj.getClass()) {
 844                 return false;
 845             }
 846             DescriptionElementCompareWrapper other = (DescriptionElementCompareWrapper) obj;
 847             if (!getOuterType().equals(other.getOuterType())) {
 848                 return false;
 849             }
 850             if (avgs == null) {
 851                 if (other.avgs != null) {
 852                     return false;
 853                 }
 854             } else if (!avgs.equals(other.avgs)) {
 855                 return false;
 856             }
 857             if (element == null) {
 858                 if (other.element != null) {
 859                     return false;
 860                 }
 861             } else if (!element.equals(other.element)) {
 862                 return false;
 863             }
 864             if (exacts == null) {
 865                 if (other.exacts != null) {
 866                     return false;
 867                 }
 868             } else if (!exacts.equals(other.exacts)) {
 869                 return false;
 870             }
 871             if (lowerBounds == null) {
 872                 if (other.lowerBounds != null) {
 873                     return false;
 874                 }
 875             } else if (!lowerBounds.equals(other.lowerBounds)) {
 876                 return false;
 877             }
 878             if (maxs == null) {
 879                 if (other.maxs != null) {
 880                     return false;
 881                 }
 882             } else if (!maxs.equals(other.maxs)) {
 883                 return false;
 884             }
 885             if (mins == null) {
 886                 if (other.mins != null) {
 887                     return false;
 888                 }
 889             } else if (!mins.equals(other.mins)) {
 890                 return false;
 891             }
 892             if (sampleSizes == null) {
 893                 if (other.sampleSizes != null) {
 894                     return false;
 895                 }
 896             } else if (!sampleSizes.equals(other.sampleSizes)) {
 897                 return false;
 898             }
 899             if (standardDevs == null) {
 900                 if (other.standardDevs != null) {
 901                     return false;
 902                 }
 903             } else if (!standardDevs.equals(other.standardDevs)) {
 904                 return false;
 905             }
 906             if (stateUuids == null) {
 907                 if (other.stateUuids != null) {
 908                     return false;
 909                 }
 910             } else if (!stateUuids.equals(other.stateUuids)) {
 911                 return false;
 912             }
 913             if (upperBounds == null) {
 914                 if (other.upperBounds != null) {
 915                     return false;
 916                 }
 917             } else if (!upperBounds.equals(other.upperBounds)) {
 918                 return false;
 919             }
 920             if (variances == null) {
 921                 if (other.variances != null) {
 922                     return false;
 923                 }
 924             } else if (!variances.equals(other.variances)) {
 925                 return false;
 926             }
 927             return true;
 928         }
 929
 930         private DescriptiveDataSetService getOuterType() {
 931             return DescriptiveDataSetService.this;
 932         }
 933
 934     }
 935
 936 }