Project

General

Profile

Revision dead0aaf

IDdead0aafea27049877dbd0fff7fcab379d57d8c1
Parent 77f78293
Child 3de07382

Added by Patrick Plitzner about 1 year ago

ref #7597 Add configuration to data set aggregation

  • implement recursive aggregation

View differences:

cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/longrunningService/ILongRunningTasksService.java
13 13

  
14 14
import eu.etaxonomy.cdm.api.service.IDescriptiveDataSetService;
15 15
import eu.etaxonomy.cdm.api.service.config.CacheUpdaterConfigurator;
16
import eu.etaxonomy.cdm.api.service.config.DescriptionAggregationConfiguration;
16 17
import eu.etaxonomy.cdm.api.service.config.ForSubtreeConfiguratorBase;
17 18
import eu.etaxonomy.cdm.api.service.config.SortIndexUpdaterConfigurator;
18 19
import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
......
51 52
    /**
52 53
     * Monitored invocation of {@link IDescriptiveDataSetService#aggregate(UUID, IProgressMonitor)}
53 54
     * @param descriptiveDataSetUuid the data set which should be aggregated
55
     * @param config the aggregation configuration
54 56
     * @return the uuid of the monitor
55 57
     */
56
    public UUID aggregateDescriptiveDataSet(UUID descriptiveDataSetUuid);
58
    public UUID aggregateDescriptiveDataSet(UUID descriptiveDataSetUuid,  DescriptionAggregationConfiguration config);
57 59

  
58 60
    /**
59 61
     * Monitored invocation of {@link IDescriptiveDataSetService#getRowWrapper(DescriptiveDataSet, IProgressMonitor)}
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/longrunningService/LongRunningTasksServiceImpl.java
21 21
import eu.etaxonomy.cdm.api.service.ITaxonNodeService;
22 22
import eu.etaxonomy.cdm.api.service.UpdateResult;
23 23
import eu.etaxonomy.cdm.api.service.config.CacheUpdaterConfigurator;
24
import eu.etaxonomy.cdm.api.service.config.DescriptionAggregationConfiguration;
24 25
import eu.etaxonomy.cdm.api.service.config.ForSubtreeConfiguratorBase;
25 26
import eu.etaxonomy.cdm.api.service.config.PublishForSubtreeConfigurator;
26 27
import eu.etaxonomy.cdm.api.service.config.SecundumForSubtreeConfigurator;
......
69 70
    }
70 71

  
71 72
    @Override
72
    public UUID aggregateDescriptiveDataSet(UUID descriptiveDataSetUuid){
73
    public UUID aggregateDescriptiveDataSet(UUID descriptiveDataSetUuid,  DescriptionAggregationConfiguration config){
73 74
        RemotingProgressMonitorThread monitorThread = new RemotingProgressMonitorThread() {
74 75
            @Override
75 76
            public Serializable doRun(IRemotingProgressMonitor monitor) {
76
                UpdateResult updateResult = descriptiveDataSetService.aggregate(descriptiveDataSetUuid, monitor);
77
                UpdateResult updateResult = descriptiveDataSetService.aggregate(descriptiveDataSetUuid, config, monitor);
77 78
                for(Exception e : updateResult.getExceptions()) {
78 79
                    monitor.addReport(e.getMessage());
79 80
                }
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/DescriptiveDataSetService.java
18 18
import org.springframework.transaction.annotation.Transactional;
19 19

  
20 20
import eu.etaxonomy.cdm.api.service.UpdateResult.Status;
21
import eu.etaxonomy.cdm.api.service.config.DescriptionAggregationConfiguration;
21 22
import eu.etaxonomy.cdm.api.service.dto.RowWrapperDTO;
22 23
import eu.etaxonomy.cdm.api.service.dto.SpecimenRowWrapperDTO;
23 24
import eu.etaxonomy.cdm.api.service.dto.TaxonRowWrapperDTO;
......
178 179
        return taxonNode;
179 180
    }
180 181

  
181
    private TaxonNode findTaxonNodeForSpecimen(TaxonNode taxonNode, SpecimenOrObservationBase specimen){
182
    private TaxonNode findTaxonNodeForSpecimen(TaxonNode taxonNode, SpecimenOrObservationBase<?> specimen){
182 183
        Collection<SpecimenNodeWrapper> nodeWrapper = occurrenceService.listUuidAndTitleCacheByAssociatedTaxon(Arrays.asList(taxonNode.getUuid()), null, null);
183 184
        for (SpecimenNodeWrapper specimenNodeWrapper : nodeWrapper) {
184 185
            if(specimenNodeWrapper.getUuidAndTitleCache().getId().equals(specimen.getId())){
......
281 282

  
282 283
    @Override
283 284
    @Transactional(readOnly=false)
284
    public UpdateResult aggregate(UUID descriptiveDataSetUuid, IProgressMonitor monitor) {
285
    public UpdateResult aggregate(UUID descriptiveDataSetUuid, DescriptionAggregationConfiguration config, IProgressMonitor monitor) {
285 286
        DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
286 287
        Set<DescriptionBase> descriptions = dataSet.getDescriptions();
287 288

  
......
290 291
        UpdateResult result = new UpdateResult();
291 292

  
292 293
        // sort descriptions by taxa
293
        Map<UUID, List<UUID>> taxaToSpecimenDescriptionMap = new HashMap<>();
294
        Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap = new HashMap<>();
294 295
        for (DescriptionBase descriptionBase : descriptions) {
295 296
            if(monitor.isCanceled()){
296 297
                result.setAbort();
......
301 302
                SpecimenDescription specimenDescription = HibernateProxyHelper.deproxy(descriptionBase, SpecimenDescription.class);
302 303
                TaxonNode taxonNode = findTaxonNodeForDescription(specimenDescription, dataSet);
303 304
                if(taxonNode!=null){
304
                    UUID taxonUuid = taxonNode.getTaxon().getUuid();
305
                    List<UUID> specimenDescriptionUuids = taxaToSpecimenDescriptionMap.get(taxonUuid);
306
                    if(specimenDescriptionUuids==null){
307
                        specimenDescriptionUuids = new ArrayList<>();
308
                    }
309
                    specimenDescriptionUuids.add(specimenDescription.getUuid());
310
                    taxaToSpecimenDescriptionMap.put(taxonUuid, specimenDescriptionUuids);
305
                    addDescriptionToTaxonNodeMap(specimenDescription.getUuid(), taxonNode, taxonNodeToSpecimenDescriptionMap);
311 306
                }
312 307
            }
313 308
            monitor.worked(1);
314 309
        }
310
        if(config.isRecursiveAggregation()){
311
            propagateDescriptionsToParentNodes(dataSet, taxonNodeToSpecimenDescriptionMap);
312
        }
315 313
        // aggregate per taxa
316
        for (Entry<UUID, List<UUID>> entry: taxaToSpecimenDescriptionMap.entrySet()) {
314
        for (Entry<TaxonNode, Set<UUID>> entry: taxonNodeToSpecimenDescriptionMap.entrySet()) {
317 315
            if(monitor.isCanceled()){
318 316
                result.setAbort();
319 317
                return result;
320 318
            }
321
            UUID taxonUuid = entry.getKey();
322
            List<UUID> specimenDescriptionUuids = entry.getValue();
323
            result.includeResult(aggregateDescription(taxonUuid , specimenDescriptionUuids, descriptiveDataSetUuid, monitor));
319
            UUID taxonUuid = entry.getKey().getTaxon().getUuid();
320
            Set<UUID> specimenDescriptionUuids = entry.getValue();
321
            result.includeResult(aggregateDescription(taxonUuid , specimenDescriptionUuids, descriptiveDataSetUuid));
324 322
            monitor.worked(1);
325 323
        }
324
        monitor.done();
326 325
        return result;
327 326
    }
328 327

  
328
    private void addDescriptionToTaxonNodeMap(UUID descriptionUuid, TaxonNode taxonNode, Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap){
329
        Set<UUID> specimenDescriptionUuids = taxonNodeToSpecimenDescriptionMap.get(taxonNode);
330
        if(specimenDescriptionUuids==null){
331
            specimenDescriptionUuids = new HashSet<>();
332
        }
333
        specimenDescriptionUuids.add(descriptionUuid);
334
        taxonNodeToSpecimenDescriptionMap.put(taxonNode, specimenDescriptionUuids);
335
    }
336

  
337
    private void propagateDescriptionsToParentNodes(DescriptiveDataSet dataSet, Map<TaxonNode, Set<UUID>> taxonNodeToSpecimenDescriptionMap){
338
        Map<TaxonNode, Set<UUID>> parentMap = new HashMap<>();
339
        for (Entry<TaxonNode, Set<UUID>> entry: taxonNodeToSpecimenDescriptionMap.entrySet()) {
340
            Set<UUID> descriptionUuids = entry.getValue();
341
            TaxonNode node = entry.getKey();
342
            TaxonNode parentNode = node.getParent();
343
            while(parentNode!=null && isTaxonNodeInDescriptiveDataSet(parentNode, dataSet)){
344
                for (UUID uuid : descriptionUuids) {
345
                    addDescriptionToTaxonNodeMap(uuid, node.getParent(), parentMap);
346
                }
347
                parentNode = parentNode.getParent();
348
            }
349
        }
350
        // merge parent map
351
        for (Entry<TaxonNode, Set<UUID>> entry: parentMap.entrySet()) {
352
            Set<UUID> descriptionUuids = entry.getValue();
353
            TaxonNode node = entry.getKey();
354
            for (UUID uuid : descriptionUuids) {
355
                addDescriptionToTaxonNodeMap(uuid, node, taxonNodeToSpecimenDescriptionMap);
356
            }
357
        }
358
    }
359

  
360
    private boolean isTaxonNodeInDescriptiveDataSet(TaxonNode taxonNode, DescriptiveDataSet dataSet){
361
        Set<TaxonNode> taxonSubtreeFilter = dataSet.getTaxonSubtreeFilter();
362
        for (TaxonNode datasetNode : taxonSubtreeFilter) {
363
            if(datasetNode.getUuid().equals(taxonNode.getUuid())){
364
                return true;
365
            }
366
            List<TaxonNode> allChildren = taxonNodeService.loadChildNodesOfTaxonNode(datasetNode, null, true, true, null);
367
            for (TaxonNode childNode : allChildren) {
368
                if(childNode.getUuid().equals(taxonNode.getUuid())){
369
                    return true;
370
                }
371
            }
372
        }
373
        return false;
374
    }
375

  
329 376
    @SuppressWarnings("unchecked")
330
    private UpdateResult aggregateDescription(UUID taxonUuid, List<UUID> descriptionUuids, UUID descriptiveDataSetUuid, IProgressMonitor monitor) {
377
    private UpdateResult aggregateDescription(UUID taxonUuid, Set<UUID> descriptionUuids, UUID descriptiveDataSetUuid) {
331 378
        UpdateResult result = new UpdateResult();
332 379

  
333 380
        TaxonBase taxonBase = taxonService.load(taxonUuid);
......
337 384
            return result;
338 385
        }
339 386
        Taxon taxon = (Taxon)taxonBase;
340
        List<DescriptionBase> descriptions = descriptionService.load(descriptionUuids, null);
387
        List<DescriptionBase> descriptions = descriptionService.load(new ArrayList<>(descriptionUuids), null);
341 388
        Map<Character, List<DescriptionElementBase>> featureToElementMap = new HashMap<>();
342 389

  
343 390
        DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
......
404 451
        return result;
405 452
    }
406 453

  
407
    private UpdateResult aggregateDescription(Taxon taxon, List<? extends DescriptionBase> descriptions, UUID descriptiveDataSetUuid) {
408
        UpdateResult result = new UpdateResult();
409
        Map<Character, List<DescriptionElementBase>> featureToElementMap = new HashMap<>();
410

  
411
        DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
412
        if(dataSet==null){
413
            result.addException(new IllegalArgumentException("Could not find data set for uuid "+descriptiveDataSetUuid));
414
            result.setAbort();
415
            return result;
416
        }
417

  
418
        //extract all character description elements
419
        descriptions.forEach(description->{
420
            description.getElements()
421
            .stream()
422
            //filter out elements that do not have a Character as Feature
423
            .filter(element->HibernateProxyHelper.isInstanceOf(((DescriptionElementBase)element).getFeature(), Character.class))
424
            .forEach(ele->{
425
                DescriptionElementBase descriptionElement = (DescriptionElementBase)ele;
426
                List<DescriptionElementBase> list = featureToElementMap.get(descriptionElement.getFeature());
427
                if(list==null){
428
                    list = new ArrayList<>();
429
                }
430
                list.add(descriptionElement);
431
                featureToElementMap.put(HibernateProxyHelper.deproxy(descriptionElement.getFeature(), Character.class), list);
432
            });
433
        });
434

  
435
        // delete existing aggregation description, if present
436
        TaxonDescription aggregation = findTaxonDescriptionByMarkerType(dataSet, taxon, MarkerType.COMPUTED());
437
        if(aggregation!=null){
438
            removeDescription(aggregation.getUuid(), descriptiveDataSetUuid);
439
        }
440

  
441
        // create new aggregation
442
        TaxonDescription description = TaxonDescription.NewInstance(taxon);
443
        description.setTitleCache("[Aggregation] "+dataSet.getTitleCache(), true);
444
        description.addMarker(Marker.NewInstance(MarkerType.COMPUTED(), true));
445
        IdentifiableSource source = IdentifiableSource.NewInstance(OriginalSourceType.Aggregation);
446
        description.addSource(source);
447
        description.addDescriptiveDataSet(dataSet);
448

  
449
        featureToElementMap.forEach((feature, elements)->{
450
            //aggregate categorical data
451
            if(feature.isSupportsCategoricalData()){
452
                CategoricalData aggregate = CategoricalData.NewInstance(feature);
453
                elements.stream()
454
                .filter(element->element instanceof CategoricalData)
455
                .forEach(categoricalData->((CategoricalData)categoricalData).getStateData()
456
                        .forEach(stateData->aggregate.addStateData((StateData) stateData.clone())));
457
                description.addElement(aggregate);
458
            }
459
            //aggregate quantitative data
460
            else if(feature.isSupportsQuantitativeData()){
461
                QuantitativeData aggregate = QuantitativeData.NewInstance(feature);
462
                elements.stream()
463
                .filter(element->element instanceof QuantitativeData)
464
                .forEach(categoricalData->((QuantitativeData)categoricalData).getStatisticalValues()
465
                        .forEach(statisticalValue->aggregate.addStatisticalValue((StatisticalMeasurementValue) statisticalValue.clone())));
466
                description.addElement(aggregate);
467
            }
468
        });
469
        result.addUpdatedObject(taxon);
470
        result.addUpdatedObject(description);
471
        return result;
472
    }
473

  
474 454
    @Override
475 455
    @Transactional(readOnly=false)
476 456
    public DeleteResult removeDescription(UUID descriptionUuid, UUID descriptiveDataSetUuid) {
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IDescriptiveDataSetService.java
7 7
import java.util.Set;
8 8
import java.util.UUID;
9 9

  
10
import eu.etaxonomy.cdm.api.service.config.DescriptionAggregationConfiguration;
10 11
import eu.etaxonomy.cdm.api.service.dto.RowWrapperDTO;
11 12
import eu.etaxonomy.cdm.api.service.dto.SpecimenRowWrapperDTO;
12 13
import eu.etaxonomy.cdm.api.service.dto.TaxonRowWrapperDTO;
......
150 151
     * directly associated with.
151 152
     * @param descriptiveDataSetUuid the uuid of the descriptive data set to which the
152 153
     * aggregated descriptions will be added to
154
     * @param config the aggregation configuration
153 155
     * @param monitor the progress monitor
154 156
     * @return the result of the operation
155 157
     */
156
    public UpdateResult aggregate(UUID descriptiveDataSetUuid, IProgressMonitor monitor);
158
    public UpdateResult aggregate(UUID descriptiveDataSetUuid,  DescriptionAggregationConfiguration config, IProgressMonitor monitor);
157 159

  
158 160
}
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/config/DescriptionAggregationConfiguration.java
1
/**
2
* Copyright (C) 2019 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.api.service.config;
10

  
11
import java.io.Serializable;
12

  
13
/**
14
 * @author pplitzner
15
 * @since Aug 8, 2019
16
 *
17
 */
18
public class DescriptionAggregationConfiguration implements Serializable {
19
    boolean recursiveAggregation = true;
20
    boolean includeDefault = true;
21
    boolean includeLiterature = false;
22

  
23
    public boolean isRecursiveAggregation() {
24
        return recursiveAggregation;
25
    }
26
    public void setRecursiveAggregation(boolean recursiveAggregation) {
27
        this.recursiveAggregation = recursiveAggregation;
28
    }
29
    public boolean isIncludeDefault() {
30
        return includeDefault;
31
    }
32
    public void setIncludeDefault(boolean includeDefault) {
33
        this.includeDefault = includeDefault;
34
    }
35
    public boolean isIncludeLiterature() {
36
        return includeLiterature;
37
    }
38
    public void setIncludeLiterature(boolean includeLiterature) {
39
        this.includeLiterature = includeLiterature;
40
    }
41

  
42
}

Also available in: Unified diff

Add picture from clipboard (Maximum size: 40 MB)