cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/description/StructuredDescriptionAggregationTest.java

   1 /**
   2 * Copyright (C) 2019 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.api.service.description;
  10
  11 import java.io.FileNotFoundException;
  12 import java.math.BigDecimal;
  13 import java.util.List;
  14 import java.util.Map;
  15 import java.util.Set;
  16 import java.util.UUID;
  17 import java.util.function.Function;
  18 import java.util.stream.Collectors;
  19
  20 import org.apache.log4j.Logger;
  21 import org.junit.Assert;
  22 import org.junit.Before;
  23 import org.junit.Ignore;
  24 import org.junit.Test;
  25 import org.unitils.dbunit.annotation.DataSet;
  26 import org.unitils.dbunit.annotation.DataSets;
  27 import org.unitils.spring.annotation.SpringBeanByType;
  28
  29 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  30 import eu.etaxonomy.cdm.api.service.IClassificationService;
  31 import eu.etaxonomy.cdm.api.service.IDescriptionService;
  32 import eu.etaxonomy.cdm.api.service.IDescriptiveDataSetService;
  33 import eu.etaxonomy.cdm.api.service.IOccurrenceService;
  34 import eu.etaxonomy.cdm.api.service.IReferenceService;
  35 import eu.etaxonomy.cdm.api.service.ITaxonNodeService;
  36 import eu.etaxonomy.cdm.api.service.ITaxonService;
  37 import eu.etaxonomy.cdm.api.service.ITermService;
  38 import eu.etaxonomy.cdm.api.service.ITermTreeService;
  39 import eu.etaxonomy.cdm.api.service.IVocabularyService;
  40 import eu.etaxonomy.cdm.api.service.UpdateResult;
  41 import eu.etaxonomy.cdm.common.JvmLimitsException;
  42 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  43 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  44 import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
  45 import eu.etaxonomy.cdm.model.common.CdmBase;
  46 import eu.etaxonomy.cdm.model.description.CategoricalData;
  47 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  48 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  49 import eu.etaxonomy.cdm.model.description.DescriptionType;
  50 import eu.etaxonomy.cdm.model.description.DescriptiveDataSet;
  51 import eu.etaxonomy.cdm.model.description.Feature;
  52 import eu.etaxonomy.cdm.model.description.IDescribable;
  53 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  54 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  55 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  56 import eu.etaxonomy.cdm.model.description.State;
  57 import eu.etaxonomy.cdm.model.description.StateData;
  58 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  59 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  60 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  61 import eu.etaxonomy.cdm.model.name.IBotanicalName;
  62 import eu.etaxonomy.cdm.model.name.Rank;
  63 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  64 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  65 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  66 import eu.etaxonomy.cdm.model.reference.Reference;
  67 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  68 import eu.etaxonomy.cdm.model.taxon.Classification;
  69 import eu.etaxonomy.cdm.model.taxon.Taxon;
  70 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  71 import eu.etaxonomy.cdm.model.term.TermNode;
  72 import eu.etaxonomy.cdm.model.term.TermTree;
  73 import eu.etaxonomy.cdm.model.term.TermType;
  74 import eu.etaxonomy.cdm.model.term.TermVocabulary;
  75 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  76 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  77
  78 /**
  79  * @author a.mueller
  80  * @since 21.11.2019
  81  */
  82 @Ignore //preliminary ignore as it does not always work (depending on other tests)
  83 public class StructuredDescriptionAggregationTest extends CdmTransactionalIntegrationTest {
  84
  85     @SuppressWarnings("unused")
  86     private static Logger logger = Logger.getLogger(StructuredDescriptionAggregationTest.class);
  87
  88     private static final UUID T_LAPSANA_UUID = UUID.fromString("f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8");
  89     private static final UUID TN_LAPSANA_UUID = UUID.fromString("f4d29e9f-6484-4184-af2e-9704e96a17e3");
  90
  91     private static final UUID T_LAPSANA_COMMUNIS_UUID = UUID.fromString("2a5ceebb-4830-4524-b330-78461bf8cb6b");
  92
  93     private static final UUID T_LAPSANA_COMMUNIS_COMMUNIS_UUID = UUID.fromString("441a3c40-0c84-11de-8c30-0800200c9a66");
  94
  95     private static final UUID T_LAPSANA_COMMUNIS_ADENOPHORA_UUID = UUID.fromString("e4acf200-63b6-11dd-ad8b-0800200c9a66");
  96
  97     private static final UUID T_LAPSANA_COMMUNIS_ALPINA_UUID = UUID.fromString("596b1325-be50-4b0a-9aa2-3ecd610215f2");
  98
  99     private static final UUID CLASSIFICATION_UUID = UUID.fromString("4b266053-a841-4980-b548-3f21d8d7d712");
 100
 101     private static final UUID T_LAPSANA_COMMUNIS_ALPINA_SPEC1_UUID = UUID.fromString("0de17403-aeef-4138-a220-9914e7c46f5a");
 102     private static final UUID T_LAPSANA_COMMUNIS_ALPINA_SPEC2_UUID = UUID.fromString("9e680e26-9e12-47a9-807c-743525c9171e");
 103     private static final UUID T_LAPSANA_COMMUNIS_ALPINA_SPEC3_UUID = UUID.fromString("3cdaa12f-7508-4073-b8d7-afca77a6be34");
 104     private static final UUID T_LAPSANA_COMMUNIS_ADENOPHORA_SPEC1_UUID = UUID.fromString("83788037-7a03-4e46-98ca-e8801096b216");
 105
 106
 107     private static final UUID uuidFeatureLeafPA = UUID.fromString("c4dfd16f-f2ed-45e0-8f4d-7fe1ae880510");
 108     private static UUID uuidFeatureLeafLength = UUID.fromString("3c19b50b-4a8e-467e-b7d4-89ebc05a33e1");
 109     private static UUID uuidFeatureLeafColor = UUID.fromString("1e8f503c-5aeb-4788-b4f9-84128f7141c7");
 110
 111     private static UUID uuidLeafColorBlue = UUID.fromString("9b4df19d-f89d-4788-9d71-d1f6f7cae910");
 112     private static UUID uuidLeafColorYellow = UUID.fromString("4cf0881b-0e7b-489a-9fdb-adbe6ae4e0ae");
 113
 114     private static UUID uuidFeatureTree = UUID.fromString("c8a29a94-2754-4d78-9faa-dff3e1387b2d");
 115
 116
 117     @SpringBeanByType
 118     private ICdmRepository repository;
 119
 120     @SpringBeanByType
 121     private ITermService termService;
 122
 123     @SpringBeanByType
 124     private ITermTreeService termTreeService;
 125
 126     @SpringBeanByType
 127     private IVocabularyService vocabularyService;
 128
 129     @SpringBeanByType
 130     private IDescriptionService descriptionService;
 131
 132     @SpringBeanByType
 133     private ITaxonService taxonService;
 134
 135     @SpringBeanByType
 136     private IOccurrenceService occurrenceService;
 137
 138     @SpringBeanByType
 139     private ITaxonNodeService taxonNodeService;
 140
 141     @SpringBeanByType
 142     private IClassificationService classificationService;
 143
 144     @SpringBeanByType
 145     private IReferenceService referenceService;
 146
 147     @SpringBeanByType
 148     private IDescriptiveDataSetService datasetService;
 149
 150     private StructuredDescriptionAggregation engine;
 151
 152     private IProgressMonitor monitor;
 153
 154     @Before
 155     public void setUp() {
 156         engine = new StructuredDescriptionAggregation();
 157 //        engine.setBatchMinFreeHeap(100 * 1024 * 1024);
 158         monitor = DefaultProgressMonitor.NewInstance();
 159     }
 160
 161     @Test
 162     @DataSets({
 163         @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="/eu/etaxonomy/cdm/database/ClearDB_with_Terms_DataSet.xml"),
 164         @DataSet(value="/eu/etaxonomy/cdm/database/TermsDataSet-with_auditing_info.xml"),
 165         @DataSet(value="StructuredDescriptionAggregationTest.xml"),
 166     })
 167     public void reaggregationTest() throws JvmLimitsException{
 168         createDefaultFeatureTree();
 169         DescriptiveDataSet dataSet = createTestDataset();
 170         commitAndStartNewTransaction();
 171
 172         StructuredDescriptionAggregationConfiguration config = createConfig(dataSet);
 173
 174         // 1st aggregation
 175         UpdateResult result = engine.invoke(config, repository);
 176         testStatusOk(result);
 177         testAggregatedDescription(false, false);
 178         addSomeDataToFirstAggregation();
 179         commitAndStartNewTransaction();
 180         testAggregatedDescription(true, false);
 181
 182         // 2nd aggregation
 183         result = engine.invoke(config, repository);
 184         testStatusOk(result);
 185         testAggregatedDescription(false, false);
 186     }
 187
 188     private void addSomeDataToFirstAggregation() {
 189         Taxon taxLapsanaCommunisAlpina = (Taxon)taxonService.find(T_LAPSANA_COMMUNIS_ALPINA_UUID);
 190         TaxonDescription taxonDescription = taxLapsanaCommunisAlpina.getDescriptions().stream()
 191                 .filter(desc->desc.getTypes().contains(DescriptionType.AGGREGATED_STRUC_DESC))
 192                 .collect(Collectors.toList()).iterator().next();
 193
 194         addCategoricalData(taxonDescription, uuidFeatureLeafPA, State.uuidPresent);
 195     }
 196
 197     @Test
 198     @DataSets({
 199         @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="/eu/etaxonomy/cdm/database/ClearDB_with_Terms_DataSet.xml"),
 200         @DataSet(value="/eu/etaxonomy/cdm/database/TermsDataSet-with_auditing_info.xml"),
 201         @DataSet(value="StructuredDescriptionAggregationTest.xml"),
 202     })
 203     public void incompleteQuantitativeDataTest() throws JvmLimitsException{
 204         createDefaultFeatureTree();
 205         DescriptiveDataSet dataSet = DescriptiveDataSet.NewInstance();
 206         datasetService.save(dataSet);
 207
 208         SpecimenDescription specDescAlpina1 = createSpecimenDescription(dataSet, T_LAPSANA_COMMUNIS_ALPINA_UUID, "alpina specimen1", T_LAPSANA_COMMUNIS_ALPINA_SPEC1_UUID);
 209         addQuantitativeData(specDescAlpina1, uuidFeatureLeafLength, StatisticalMeasure.MIN(), new BigDecimal("5.0"));
 210
 211         SpecimenDescription specDescAlpina2 = createSpecimenDescription(dataSet, T_LAPSANA_COMMUNIS_ALPINA_UUID, "alpina specimen2", T_LAPSANA_COMMUNIS_ALPINA_SPEC2_UUID);
 212         addQuantitativeData(specDescAlpina2, uuidFeatureLeafLength, StatisticalMeasure.MAX(), new BigDecimal("7.0"));
 213
 214         TaxonNode tnLapsana = taxonNodeService.find(TN_LAPSANA_UUID);
 215         Assert.assertNotNull(tnLapsana);
 216         dataSet.addTaxonSubtree(tnLapsana);
 217
 218         @SuppressWarnings("unchecked")
 219         TermTree<Feature> descriptiveSystem = termTreeService.find(uuidFeatureTree);
 220         dataSet.setDescriptiveSystem(descriptiveSystem);
 221         commitAndStartNewTransaction();
 222
 223         StructuredDescriptionAggregationConfiguration config = createConfig(dataSet);
 224
 225         UpdateResult result = engine.invoke(config, repository);
 226         testStatusOk(result);
 227
 228         Taxon taxLapsanaCommunisAlpina = (Taxon)taxonService.find(T_LAPSANA_COMMUNIS_ALPINA_UUID);
 229         TaxonDescription aggrDescLapsanaCommunisAlpina = testTaxonDescriptions(taxLapsanaCommunisAlpina, 1);
 230         testQuantitativeData(uuidFeatureLeafLength, null, new BigDecimal("0.0"), new BigDecimal("7.0"), null, aggrDescLapsanaCommunisAlpina);
 231     }
 232
 233     @Test
 234     @DataSets({
 235         @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="/eu/etaxonomy/cdm/database/ClearDB_with_Terms_DataSet.xml"),
 236         @DataSet(value="/eu/etaxonomy/cdm/database/TermsDataSet-with_auditing_info.xml"),
 237         @DataSet(value="StructuredDescriptionAggregationTest.xml"),
 238     })
 239     public void incompleteCategoricalDataTest() throws JvmLimitsException{
 240         createDefaultFeatureTree();
 241         DescriptiveDataSet dataSet = DescriptiveDataSet.NewInstance();
 242         datasetService.save(dataSet);
 243
 244         SpecimenDescription specDescAlpina1 = createSpecimenDescription(dataSet, T_LAPSANA_COMMUNIS_ALPINA_UUID, "alpina specimen1", T_LAPSANA_COMMUNIS_ALPINA_SPEC1_UUID);
 245         addCategoricalData(specDescAlpina1, uuidFeatureLeafColor, null);
 246
 247         TaxonNode tnLapsana = taxonNodeService.find(TN_LAPSANA_UUID);
 248         Assert.assertNotNull(tnLapsana);
 249         dataSet.addTaxonSubtree(tnLapsana);
 250
 251         @SuppressWarnings("unchecked")
 252         TermTree<Feature> descriptiveSystem = termTreeService.find(uuidFeatureTree);
 253         dataSet.setDescriptiveSystem(descriptiveSystem);
 254         commitAndStartNewTransaction();
 255
 256         StructuredDescriptionAggregationConfiguration config = createConfig(dataSet);
 257
 258         UpdateResult result = engine.invoke(config, repository);
 259         testStatusOk(result);
 260
 261         Taxon taxLapsanaCommunisAlpina = (Taxon)taxonService.find(T_LAPSANA_COMMUNIS_ALPINA_UUID);
 262         TaxonDescription aggrDescLapsanaCommunisAlpina = testTaxonDescriptions(taxLapsanaCommunisAlpina, 1);
 263         List<StateData> sdAlpinaLeafColor = testCategoricalData(uuidFeatureLeafColor, 1, aggrDescLapsanaCommunisAlpina, false);
 264         testState(sdAlpinaLeafColor, uuidLeafColorBlue, 0);
 265         testState(sdAlpinaLeafColor, uuidLeafColorYellow, 0);
 266     }
 267
 268     @Test
 269     @DataSets({
 270         @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="/eu/etaxonomy/cdm/database/ClearDB_with_Terms_DataSet.xml"),
 271         @DataSet(value="/eu/etaxonomy/cdm/database/TermsDataSet-with_auditing_info.xml"),
 272         @DataSet(value="StructuredDescriptionAggregationTest.xml"),
 273     })
 274     public void aggregationTest() throws JvmLimitsException{
 275         createDefaultFeatureTree();
 276         DescriptiveDataSet dataSet = createTestDataset();
 277         commitAndStartNewTransaction();
 278
 279         StructuredDescriptionAggregationConfiguration config = createConfig(dataSet);
 280
 281         UpdateResult result = engine.invoke(config, repository);
 282         commitAndStartNewTransaction();
 283         testStatusOk(result);
 284         testAggregatedDescription(false, false);
 285
 286         config.setIncludeLiterature(true);
 287
 288         result = engine.invoke(config, repository);
 289         commitAndStartNewTransaction();
 290         testStatusOk(result);
 291         testAggregatedDescription(false, true);  //with literature
 292     }
 293
 294     private void testStatusOk(UpdateResult result) {
 295         if (result.getStatus() != UpdateResult.Status.OK){
 296             Assert.fail("Aggregation should have status OK but was " + result.toString());
 297             for (Exception ex : result.getExceptions()){
 298                 ex.printStackTrace();
 299             }
 300         }
 301     }
 302
 303     private void addLiterature(DescriptiveDataSet dataSet) {
 304
 305         //literature description
 306         Taxon taxon = (Taxon)taxonService.find(T_LAPSANA_COMMUNIS_ALPINA_UUID);
 307         TaxonDescription literatureDescription = TaxonDescription.NewInstance(taxon);
 308         literatureDescription.addType(DescriptionType.SECONDARY_DATA);
 309         addQuantitativeData(literatureDescription, uuidFeatureLeafLength, new BigDecimal("4.5"), new BigDecimal("6.5"));
 310         addCategoricalData(literatureDescription, uuidFeatureLeafColor, uuidLeafColorBlue);
 311         dataSet.addDescription(literatureDescription);
 312     }
 313
 314     private void testAggregatedDescription(boolean withAddedData, boolean withLiterature) {
 315
 316         //L. communis alpina
 317         Taxon taxLapsanaCommunisAlpina = (Taxon)taxonService.find(T_LAPSANA_COMMUNIS_ALPINA_UUID);
 318         int nElement = withAddedData ? 4 : 3;
 319         TaxonDescription aggrDescLapsanaCommunisAlpina = testTaxonDescriptions(taxLapsanaCommunisAlpina, nElement);
 320
 321         List<StateData> stateData = testCategoricalData(uuidFeatureLeafPA, 1, aggrDescLapsanaCommunisAlpina, withAddedData);
 322         testState(stateData, State.uuidPresent, 3);
 323         List<StateData> sdAlpinaLeafColor = testCategoricalData(uuidFeatureLeafColor, 1, aggrDescLapsanaCommunisAlpina, false);
 324         int litLeafColorBlue = withLiterature? 1: 0;
 325         testState(sdAlpinaLeafColor, uuidLeafColorBlue, 2+litLeafColorBlue);
 326         testState(sdAlpinaLeafColor, uuidLeafColorYellow, 0);
 327         BigDecimal count = withLiterature? null : new BigDecimal("3");
 328         BigDecimal avg = withLiterature? null : new BigDecimal("6.666667");
 329         BigDecimal min = withLiterature? new BigDecimal("4.5") : new BigDecimal("5.0");
 330         testQuantitativeData(uuidFeatureLeafLength, count, min,
 331                 new BigDecimal("8.0"), avg, aggrDescLapsanaCommunisAlpina);
 332         //... sources
 333         int intLit = withLiterature? 1 : 0;
 334         Assert.assertEquals(3+intLit, aggrDescLapsanaCommunisAlpina.getSources().size());
 335         SpecimenOrObservationBase<?> specLcommunisAlpina1 = CdmBase.deproxy(occurrenceService.find(T_LAPSANA_COMMUNIS_ALPINA_SPEC1_UUID));
 336         Assert.assertEquals("Spec1 must have 2 descriptions now. The primary one and the cloned.", 2, specLcommunisAlpina1.getSpecimenDescriptions().size());
 337         Assert.assertEquals(1, specLcommunisAlpina1.getSpecimenDescriptions().stream().filter(d->d.isCloneForSource()).count());
 338         DescriptionBase<?> clonedDesc = specLcommunisAlpina1.getDescriptions().stream().filter(d->d.isCloneForSource()).findFirst().get();
 339         DescriptionBase<?> sourceDescription = getSingleSpecimenDescriptionSource(aggrDescLapsanaCommunisAlpina, T_LAPSANA_COMMUNIS_ALPINA_SPEC1_UUID);
 340         Assert.assertEquals(clonedDesc, sourceDescription);
 341
 342         //L. communis adenophora
 343         Taxon taxLapsanaCommunisAdenophora = (Taxon)taxonService.find(T_LAPSANA_COMMUNIS_ADENOPHORA_UUID);
 344         TaxonDescription aggrDescLapsanaCommunisAdenophora = testTaxonDescriptions(taxLapsanaCommunisAdenophora, 3);
 345         testState(testCategoricalData(uuidFeatureLeafPA, 1, aggrDescLapsanaCommunisAdenophora, false), State.uuidPresent, 1);
 346         List<StateData> sdAdenophoraLeafColor = testCategoricalData(uuidFeatureLeafColor, 1, aggrDescLapsanaCommunisAdenophora, false);
 347         testState(sdAdenophoraLeafColor, uuidLeafColorBlue, 0);
 348         testState(sdAdenophoraLeafColor, uuidLeafColorYellow, 1);
 349         testQuantitativeData(uuidFeatureLeafLength, new BigDecimal("1"), new BigDecimal("10.0"),
 350                 new BigDecimal("10.0"), new BigDecimal("10.0"), aggrDescLapsanaCommunisAdenophora);
 351
 352         //L. communis
 353         Taxon taxLapsanaCommunis = (Taxon)taxonService.find(T_LAPSANA_COMMUNIS_UUID);
 354         TaxonDescription aggrDescLapsanaCommunis = testTaxonDescriptions(taxLapsanaCommunis, 3);
 355         testState(testCategoricalData(uuidFeatureLeafPA, 1, aggrDescLapsanaCommunis, false), State.uuidPresent, 4);
 356         List<StateData> sdCommunisLeafColor = testCategoricalData(uuidFeatureLeafColor, 2, aggrDescLapsanaCommunis, false);
 357         testState(sdCommunisLeafColor, uuidLeafColorBlue, 2 + intLit);
 358         testState(sdCommunisLeafColor, uuidLeafColorYellow, 1);
 359         count = withLiterature? null : new BigDecimal("4");
 360         avg = withLiterature? null : new BigDecimal("7.5");
 361         testQuantitativeData(uuidFeatureLeafLength, count, min,
 362                 new BigDecimal("10.0"), avg, aggrDescLapsanaCommunis);
 363         //... sources
 364         Assert.assertEquals(2, aggrDescLapsanaCommunis.getSources().size());
 365         Map<UUID, List<TaxonDescription>> taxonDescriptionMap = getSourceTaxonDescriptionMap(aggrDescLapsanaCommunis);
 366         Assert.assertEquals(2, taxonDescriptionMap.size());
 367         Assert.assertEquals(1, taxonDescriptionMap.get(T_LAPSANA_COMMUNIS_ALPINA_UUID).size());
 368         Assert.assertEquals(1, taxonDescriptionMap.get(T_LAPSANA_COMMUNIS_ADENOPHORA_UUID).size());
 369         Assert.assertNotEquals(aggrDescLapsanaCommunisAlpina, taxonDescriptionMap.get(T_LAPSANA_COMMUNIS_ALPINA_UUID).get(0));
 370
 371         //Lapsana
 372         Taxon taxLapsana = (Taxon)taxonService.find(T_LAPSANA_UUID);
 373         TaxonDescription aggrDescLapsana = testTaxonDescriptions(taxLapsana, 3);
 374         testState(testCategoricalData(uuidFeatureLeafPA, 1, aggrDescLapsana, false), State.uuidPresent, 4);
 375         List<StateData> sdLapsanLeafColor = testCategoricalData(uuidFeatureLeafColor, 2, aggrDescLapsana, false);
 376         testState(sdLapsanLeafColor, uuidLeafColorBlue, 2 + intLit);
 377         testState(sdLapsanLeafColor, uuidLeafColorYellow, 1);
 378         count = withLiterature? null : new BigDecimal("4");
 379         avg = withLiterature? null : new BigDecimal("7.5");
 380         testQuantitativeData(uuidFeatureLeafLength, count, min,
 381                 new BigDecimal("10.0"), avg, aggrDescLapsana);
 382         //... sources
 383         Assert.assertEquals(1, aggrDescLapsana.getSources().size());
 384         taxonDescriptionMap = getSourceTaxonDescriptionMap(aggrDescLapsana);
 385         Assert.assertEquals(1, taxonDescriptionMap.size());
 386         Assert.assertEquals(1, taxonDescriptionMap.get(T_LAPSANA_COMMUNIS_UUID).size());
 387         Assert.assertNotEquals(aggrDescLapsanaCommunis, taxonDescriptionMap.get(T_LAPSANA_COMMUNIS_UUID).get(0));
 388
 389     }
 390
 391     private Map<UUID, List<TaxonDescription>> getSourceTaxonDescriptionMap(TaxonDescription desc) {
 392         return desc.getSources().stream().filter(s->(s.getCdmSource() instanceof TaxonDescription))
 393             .map(s->CdmBase.deproxy(s.getCdmSource(), TaxonDescription.class))
 394             .collect(Collectors.groupingBy(fDescToDescribedUuid));
 395     }
 396
 397     private static Function<DescriptionBase<?>, UUID> fDescToDescribedUuid =
 398             ((Function<DescriptionBase<?>, IDescribable<?>>)(d->d.isInstanceOf(SpecimenDescription.class)? d.getDescribedSpecimenOrObservation(): CdmBase.deproxy(d, TaxonDescription.class).getTaxon()))
 399             .andThen(IDescribable::getUuid);
 400
 401     private DescriptionBase<?> getSingleSpecimenDescriptionSource(
 402             TaxonDescription aggrDescLapsanaCommunisAlpina, UUID specimenUuid) {
 403
 404         Map<UUID, List<DescriptionBase<?>>> map = aggrDescLapsanaCommunisAlpina.getSources().stream()
 405             .map(src->(DescriptionBase<?>)src.getCdmSource())
 406             .filter(o-> o != null)
 407             .collect(Collectors.groupingBy(fDescToDescribedUuid));
 408         Assert.assertEquals(1, map.get(specimenUuid).size());
 409         return map.get(specimenUuid).get(0);
 410
 411     }
 412
 413     private StructuredDescriptionAggregationConfiguration createConfig(DescriptiveDataSet dataSet) {
 414         TaxonNodeFilter filter = TaxonNodeFilter.NewSubtreeInstance(TN_LAPSANA_UUID);
 415         StructuredDescriptionAggregationConfiguration config =
 416                 StructuredDescriptionAggregationConfiguration.NewInstance(filter, monitor);
 417         config.setDatasetUuid(dataSet.getUuid());
 418         config.setAggregationMode(AggregationMode.byWithinTaxonAndToParent());
 419         config.setIncludeLiterature(false);
 420         return config;
 421     }
 422
 423     private DescriptiveDataSet createTestDataset() {
 424         DescriptiveDataSet dataSet = DescriptiveDataSet.NewInstance();
 425         dataSet.setLabel("Test dataset");
 426         datasetService.save(dataSet);
 427
 428         //L. communis alpina spec1
 429         SpecimenDescription specDescAlpina1 = createSpecimenDescription(dataSet, T_LAPSANA_COMMUNIS_ALPINA_UUID, "alpina specimen1", T_LAPSANA_COMMUNIS_ALPINA_SPEC1_UUID);
 430         addCategoricalData(specDescAlpina1, uuidFeatureLeafPA, State.uuidPresent);
 431         addQuantitativeData(specDescAlpina1, uuidFeatureLeafLength, StatisticalMeasure.EXACT_VALUE(), new BigDecimal("5.0"));
 432         addCategoricalData(specDescAlpina1, uuidFeatureLeafColor, uuidLeafColorBlue);
 433
 434         //L. communis alpina spec2
 435         SpecimenDescription specDescAlpina2 = createSpecimenDescription(dataSet, T_LAPSANA_COMMUNIS_ALPINA_UUID, "alpina specimen2", T_LAPSANA_COMMUNIS_ALPINA_SPEC2_UUID);
 436         addCategoricalData(specDescAlpina2, uuidFeatureLeafPA, State.uuidPresent);
 437         addQuantitativeData(specDescAlpina2, uuidFeatureLeafLength, StatisticalMeasure.EXACT_VALUE(), new BigDecimal("7.0"));
 438         addCategoricalData(specDescAlpina2, uuidFeatureLeafColor, uuidLeafColorBlue);
 439
 440         //L. communis alpina spec3
 441         SpecimenDescription specDescAlpina3 = createSpecimenDescription(dataSet, T_LAPSANA_COMMUNIS_ALPINA_UUID, "alpina specimen3", T_LAPSANA_COMMUNIS_ALPINA_SPEC3_UUID);
 442         addCategoricalData(specDescAlpina3, uuidFeatureLeafPA, State.uuidPresent);
 443         addQuantitativeData(specDescAlpina3, uuidFeatureLeafLength, StatisticalMeasure.EXACT_VALUE(), new BigDecimal("8.0"));
 444
 445         //L. communis adenophora
 446         SpecimenDescription specDescAdenophora = createSpecimenDescription(dataSet, T_LAPSANA_COMMUNIS_ADENOPHORA_UUID, "adenophora specimen", T_LAPSANA_COMMUNIS_ADENOPHORA_SPEC1_UUID);
 447         addCategoricalData(specDescAdenophora, uuidFeatureLeafPA, State.uuidPresent);
 448         addQuantitativeData(specDescAdenophora, uuidFeatureLeafLength, StatisticalMeasure.EXACT_VALUE(), new BigDecimal("10.0"));
 449         addCategoricalData(specDescAdenophora, uuidFeatureLeafColor, uuidLeafColorYellow);
 450
 451         TaxonNode tnLapsana = taxonNodeService.find(TN_LAPSANA_UUID);
 452         Assert.assertNotNull(tnLapsana);
 453         dataSet.addTaxonSubtree(tnLapsana);
 454
 455         @SuppressWarnings("unchecked")
 456         TermTree<Feature> descriptiveSystem = termTreeService.find(uuidFeatureTree);
 457         dataSet.setDescriptiveSystem(descriptiveSystem);
 458
 459         addLiterature(dataSet);
 460         return dataSet;
 461     }
 462
 463     private TaxonDescription testTaxonDescriptions(Taxon taxon, int elementSize){
 464         List<TaxonDescription> taxonDescriptions = taxon.getDescriptions().stream()
 465                 .filter(desc->desc.getTypes().contains(DescriptionType.AGGREGATED_STRUC_DESC))
 466                 .filter(desc->!desc.getTypes().contains(DescriptionType.CLONE_FOR_SOURCE))
 467                 .collect(Collectors.toList());
 468
 469         Assert.assertEquals(1, taxonDescriptions.size());
 470         TaxonDescription aggrDesc = taxonDescriptions.iterator().next();
 471         Set<DescriptionElementBase> elements = aggrDesc.getElements();
 472         Assert.assertEquals(elementSize, elements.size());
 473         return aggrDesc;
 474     }
 475
 476     private void testQuantitativeData(UUID featureUuid, BigDecimal sampleSize, BigDecimal min,
 477             BigDecimal max, BigDecimal avg, TaxonDescription aggrDesc) {
 478         List<QuantitativeData> quantitativeDatas = aggrDesc.getElements().stream()
 479                 .filter(element->element.getFeature().getUuid().equals(featureUuid))
 480                 .map(catData->CdmBase.deproxy(catData, QuantitativeData.class))
 481                 .collect(Collectors.toList());
 482         Assert.assertEquals(1, quantitativeDatas.size());
 483         QuantitativeData leafLength = quantitativeDatas.iterator().next();
 484         Assert.assertEquals(sampleSize, leafLength.getSampleSize());
 485         Assert.assertEquals(min, leafLength.getMin());
 486         Assert.assertEquals(max, leafLength.getMax());
 487         Assert.assertEquals(avg, leafLength.getAverage());
 488     }
 489
 490
 491     private List<StateData> testCategoricalData(UUID featureUuid, int stateDataCount, TaxonDescription taxonDescription, boolean withAddedData) {
 492         List<CategoricalData> categoricalDatas = taxonDescription.getElements().stream()
 493                 .filter(element->element.getFeature().getUuid().equals(featureUuid))
 494                 .map(catData->CdmBase.deproxy(catData, CategoricalData.class))
 495                 .collect(Collectors.toList());
 496         int nCD = withAddedData ? 2 : 1;
 497         Assert.assertEquals(nCD, categoricalDatas.size());
 498         CategoricalData categoricalData;
 499         if (withAddedData){
 500             categoricalData = categoricalDatas.stream().filter(cd->cd.getStateData().get(0).getCount() != null ).findFirst().get();
 501         }else{
 502             categoricalData = categoricalDatas.iterator().next(); // categoricalDatas.stream().filter(cd->cd.getStateData().size() != 1).collect(Collectors.toList());
 503         }
 504         List<StateData> stateDatas = categoricalData.getStateData();
 505         Assert.assertEquals(stateDataCount, stateDatas.size());
 506         return stateDatas;
 507     }
 508
 509     private void testState(List<StateData> stateDatas, UUID stateUuid, Integer stateDataCount){
 510         List<StateData> filteredStateDatas = stateDatas.stream()
 511                 .filter(stateData->stateData.getState()!=null && stateData.getState().getUuid().equals(stateUuid))
 512                 .collect(Collectors.toList());
 513         if(stateDataCount==0){
 514             // non-existence test
 515             Assert.assertEquals(0, filteredStateDatas.size());
 516             return;
 517         }
 518         Assert.assertEquals(1, filteredStateDatas.size());
 519         StateData stateData = filteredStateDatas.iterator().next();
 520         Assert.assertEquals(stateDataCount, stateData.getCount());
 521         Assert.assertEquals(stateUuid, stateData.getState().getUuid());
 522     }
 523
 524     private void addQuantitativeData(DescriptionBase<?> desc, UUID uuidFeature, StatisticalMeasure type, BigDecimal value) {
 525         Feature feature = (Feature)termService.find(uuidFeature);
 526         QuantitativeData qd = QuantitativeData.NewInstance(feature);
 527         StatisticalMeasurementValue smv = StatisticalMeasurementValue.NewInstance(type, value);
 528         qd.addStatisticalValue(smv);
 529         desc.addElement(qd);
 530     }
 531
 532     private void addQuantitativeData(DescriptionBase<?> desc, UUID uuidFeature, BigDecimal min, BigDecimal max) {
 533         Feature feature = (Feature)termService.find(uuidFeature);
 534         QuantitativeData qd = QuantitativeData.NewInstance(feature);
 535         StatisticalMeasurementValue smv = StatisticalMeasurementValue.NewInstance(StatisticalMeasure.MIN(), min);
 536         qd.addStatisticalValue(smv);
 537         smv = StatisticalMeasurementValue.NewInstance(StatisticalMeasure.MAX(), max);
 538         qd.addStatisticalValue(smv);
 539         desc.addElement(qd);
 540     }
 541
 542     private void addCategoricalData(DescriptionBase<?> desc, UUID featureUuid, UUID stateUUID) {
 543         Feature feature = (Feature)termService.find(featureUuid);
 544         State state = (State)termService.find(stateUUID);
 545         CategoricalData cd = CategoricalData.NewInstance(state, feature);
 546         desc.addElement(cd);
 547     }
 548
 549     private SpecimenDescription createSpecimenDescription(DescriptiveDataSet dataSet, UUID taxonUuid, String specLabel, UUID specimenUuid ) {
 550         Taxon taxon = (Taxon)taxonService.find(taxonUuid);
 551         TaxonDescription taxonDescription = TaxonDescription.NewInstance(taxon);
 552         DerivedUnit specimen = DerivedUnit.NewPreservedSpecimenInstance();
 553         specimen.setTitleCache(specLabel, true);
 554         specimen.setUuid(specimenUuid);
 555         IndividualsAssociation individualsAssociation = IndividualsAssociation.NewInstance(specimen);
 556         // TODO this has to be discussed; currently the description with the InidividualsAssociation is
 557         // needed in the dataset for performance reasons
 558         taxonDescription.addElement(individualsAssociation);
 559         dataSet.addDescription(taxonDescription);
 560         SpecimenDescription specDesc = SpecimenDescription.NewInstance(specimen);
 561
 562         dataSet.addDescription(specDesc);
 563         return specDesc;
 564     }
 565
 566     private Feature createFeature(UUID uuid, String label, boolean isQuantitative) {
 567         Feature feature = Feature.NewInstance("", label, null);
 568         feature.setUuid(uuid);
 569         feature.setSupportsQuantitativeData(isQuantitative);
 570         feature.setSupportsCategoricalData(!isQuantitative);
 571         feature.setSupportsTextData(false);
 572         termService.save(feature);
 573         return feature;
 574     }
 575
 576     private State createState(String label, UUID uuid) {
 577         State state = State.NewInstance("", label, "");
 578         state.getTitleCache();  //for better debugging
 579         state.setUuid(uuid);
 580         termService.save(state);
 581         return state;
 582     }
 583
 584     private void createDefaultFeatureTree() {
 585         //feature tree
 586         //leaf p/a
 587         //  leaf length
 588         //  leaf color
 589         TermTree<Feature> featureTree = TermTree.NewFeatureInstance();
 590         featureTree.setUuid(uuidFeatureTree);
 591         Feature featureLeafPA = createFeature(uuidFeatureLeafPA, "LeafPA", false);
 592         TermNode<Feature> leafPANode = featureTree.getRoot().addChild(featureLeafPA);
 593         Feature featureLeafLength = createFeature(uuidFeatureLeafLength, "LeafLength", true);
 594         leafPANode.addChild(featureLeafLength);
 595         Feature featureLeafColor = createFeature(uuidFeatureLeafColor, "LeafColor", false);
 596         leafPANode.addChild(featureLeafColor);
 597         State yellow = createState("Yellow", uuidLeafColorYellow);
 598         State blue = createState("Blue", uuidLeafColorBlue);
 599         TermVocabulary<State> stateVoc = TermVocabulary.NewInstance(TermType.State, State.class, "", "Colors", null, null);
 600         stateVoc.addTerm(yellow);
 601         stateVoc.addTerm(blue);
 602         featureLeafColor.addSupportedCategoricalEnumeration(stateVoc);
 603         vocabularyService.save(stateVoc);
 604     }
 605
 606 //    @Test
 607     @Override
 608     public void createTestDataSet() throws FileNotFoundException {
 609
 610         // --- References --- //
 611         Reference sec = ReferenceFactory.newDatabase();
 612         sec.setTitleCache("Test", true);
 613         Reference nomRef = ReferenceFactory.newBook();
 614         sec.setTitleCache("Sp.Pl.", true);
 615
 616         referenceService.save(sec);
 617         referenceService.save(nomRef);
 618
 619
 620         // --- Taxa --- //
 621         //  Lapsana
 622         //        L. communis
 623         //            L. communis subsp. communis
 624         //            L. communis subsp. adenophora
 625         //            L. communis subsp. alpina
 626         //  Sonchella
 627         //        S. dentata
 628         //        S. stenoma
 629         IBotanicalName n_lapsana = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
 630         n_lapsana.setTitleCache("Lapsana", true);
 631         Taxon t_lapsana = Taxon.NewInstance(n_lapsana, sec);
 632         t_lapsana.setUuid(T_LAPSANA_UUID);
 633         taxonService.saveOrUpdate(t_lapsana);
 634
 635         IBotanicalName n_lapsana_communis = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
 636         n_lapsana_communis.setTitleCache("L. communis", true);
 637         Taxon t_lapsana_communis = Taxon.NewInstance(n_lapsana_communis, sec);
 638         t_lapsana_communis.setUuid(T_LAPSANA_COMMUNIS_UUID);
 639         taxonService.saveOrUpdate(t_lapsana_communis);
 640
 641         IBotanicalName n_lapsana_communis_communis = TaxonNameFactory.NewBotanicalInstance(Rank.SUBSPECIES());
 642         n_lapsana_communis_communis.setTitleCache("L. communis subsp. communis", true);
 643         Taxon t_lapsana_communis_communis = Taxon.NewInstance(n_lapsana_communis_communis, sec);
 644         t_lapsana_communis_communis.setUuid(T_LAPSANA_COMMUNIS_COMMUNIS_UUID);
 645         taxonService.saveOrUpdate(t_lapsana_communis_communis);
 646
 647         IBotanicalName n_lapsana_communis_adenophora = TaxonNameFactory.NewBotanicalInstance(Rank.SUBSPECIES());
 648         n_lapsana_communis_adenophora.setTitleCache("L. communis subsp. adenophora", true);
 649         Taxon t_lapsana_communis_adenophora = Taxon.NewInstance(n_lapsana_communis_adenophora, sec);
 650         t_lapsana_communis_adenophora.setUuid(T_LAPSANA_COMMUNIS_ADENOPHORA_UUID);
 651         taxonService.saveOrUpdate(t_lapsana_communis_adenophora);
 652
 653         IBotanicalName n_lapsana_communis_alpina = TaxonNameFactory.NewBotanicalInstance(Rank.SUBSPECIES());
 654         n_lapsana_communis_alpina.setTitleCache("L. communis subsp. alpina", true);
 655         Taxon t_lapsana_communis_alpina = Taxon.NewInstance(n_lapsana_communis_alpina, sec);
 656         t_lapsana_communis_alpina.setUuid(T_LAPSANA_COMMUNIS_ALPINA_UUID);
 657         taxonService.saveOrUpdate(t_lapsana_communis_alpina);
 658
 659         // --- Classification --- //
 660         Classification classification = Classification.NewInstance("TestClassification");
 661         classification.setUuid(CLASSIFICATION_UUID);
 662         classificationService.save(classification);
 663         TaxonNode node_lapsana = classification.addChildTaxon(t_lapsana, sec, null);
 664         node_lapsana.setUuid(TN_LAPSANA_UUID);
 665         TaxonNode node_lapsana_communis = node_lapsana.addChildTaxon(t_lapsana_communis, sec, null);
 666         node_lapsana_communis.addChildTaxon(t_lapsana_communis_communis, sec, null);
 667         node_lapsana_communis.addChildTaxon(t_lapsana_communis_adenophora, sec, null);
 668         node_lapsana_communis.addChildTaxon(t_lapsana_communis_alpina, sec, null);
 669         classificationService.saveOrUpdate(classification);
 670
 671         commitAndStartNewTransaction(null);
 672
 673         writeDbUnitDataSetFile(new String[] {
 674                 "TAXONBASE", "TAXONNAME","CLASSIFICATION",  "TAXONNODE","HOMOTYPICALGROUP",
 675                 "REFERENCE", "AGENTBASE",
 676                 "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
 677                 "LANGUAGESTRING",
 678                 "HIBERNATE_SEQUENCES"
 679          });
 680     }
 681
 682
 683 }