cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/DescriptiveDataSetService.java

   1 package eu.etaxonomy.cdm.api.service;
   2
   3 import java.math.BigDecimal;
   4 import java.util.ArrayList;
   5 import java.util.Arrays;
   6 import java.util.Collection;
   7 import java.util.Collections;
   8 import java.util.HashMap;
   9 import java.util.HashSet;
  10 import java.util.List;
  11 import java.util.Map;
  12 import java.util.Optional;
  13 import java.util.Set;
  14 import java.util.UUID;
  15 import java.util.stream.Collectors;
  16
  17 import org.apache.log4j.Logger;
  18 import org.springframework.beans.factory.annotation.Autowired;
  19 import org.springframework.stereotype.Service;
  20 import org.springframework.transaction.annotation.Transactional;
  21
  22 import eu.etaxonomy.cdm.api.service.UpdateResult.Status;
  23 import eu.etaxonomy.cdm.api.service.config.DeleteDescriptiveDataSetConfigurator;
  24 import eu.etaxonomy.cdm.api.service.config.IdentifiableServiceConfiguratorImpl;
  25 import eu.etaxonomy.cdm.api.service.config.RemoveDescriptionsFromDescriptiveDataSetConfigurator;
  26 import eu.etaxonomy.cdm.api.service.dto.CategoricalDataDto;
  27 import eu.etaxonomy.cdm.api.service.dto.DescriptionBaseDto;
  28 import eu.etaxonomy.cdm.api.service.dto.DescriptionElementDto;
  29 import eu.etaxonomy.cdm.api.service.dto.QuantitativeDataDto;
  30 import eu.etaxonomy.cdm.api.service.dto.RowWrapperDTO;
  31 import eu.etaxonomy.cdm.api.service.dto.SpecimenOrObservationDTOFactory;
  32 import eu.etaxonomy.cdm.api.service.dto.SpecimenRowWrapperDTO;
  33 import eu.etaxonomy.cdm.api.service.dto.StateDataDto;
  34 import eu.etaxonomy.cdm.api.service.dto.StatisticalMeasurementValueDto;
  35 import eu.etaxonomy.cdm.api.service.dto.TaxonRowWrapperDTO;
  36 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  37 import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
  38 import eu.etaxonomy.cdm.format.description.DefaultCategoricalDescriptionBuilder;
  39 import eu.etaxonomy.cdm.format.description.DefaultQuantitativeDescriptionBuilder;
  40 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  41 import eu.etaxonomy.cdm.model.common.CdmBase;
  42 import eu.etaxonomy.cdm.model.common.Language;
  43 import eu.etaxonomy.cdm.model.description.CategoricalData;
  44 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  45 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  46 import eu.etaxonomy.cdm.model.description.DescriptionType;
  47 import eu.etaxonomy.cdm.model.description.DescriptiveDataSet;
  48 import eu.etaxonomy.cdm.model.description.DescriptiveSystemRole;
  49 import eu.etaxonomy.cdm.model.description.Feature;
  50 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  51 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  52 import eu.etaxonomy.cdm.model.description.PolytomousKey;
  53 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  54 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  55 import eu.etaxonomy.cdm.model.description.State;
  56 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  57 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  58 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  59 import eu.etaxonomy.cdm.model.description.TextData;
  60 import eu.etaxonomy.cdm.model.location.NamedArea;
  61 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  62 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  63 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  64 import eu.etaxonomy.cdm.model.reference.CdmLinkSource;
  65 import eu.etaxonomy.cdm.model.taxon.Classification;
  66 import eu.etaxonomy.cdm.model.taxon.Taxon;
  67 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  68 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  69 import eu.etaxonomy.cdm.model.term.DefinedTermBase;
  70 import eu.etaxonomy.cdm.persistence.dao.description.IDescriptiveDataSetDao;
  71 import eu.etaxonomy.cdm.persistence.dao.term.IDefinedTermDao;
  72 import eu.etaxonomy.cdm.persistence.dto.DescriptiveDataSetBaseDto;
  73 import eu.etaxonomy.cdm.persistence.dto.MergeResult;
  74 import eu.etaxonomy.cdm.persistence.dto.SpecimenNodeWrapper;
  75 import eu.etaxonomy.cdm.persistence.dto.TaxonNodeDto;
  76 import eu.etaxonomy.cdm.persistence.dto.TermDto;
  77 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  78 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
  79 import eu.etaxonomy.cdm.strategy.generate.PolytomousKeyGenerator;
  80 import eu.etaxonomy.cdm.strategy.generate.PolytomousKeyGeneratorConfigurator;
  81
  82 @Service
  83 @Transactional(readOnly=true)
  84 public class DescriptiveDataSetService
  85         extends IdentifiableServiceBase<DescriptiveDataSet, IDescriptiveDataSetDao>
  86         implements IDescriptiveDataSetService {
  87
  88     private static Logger logger = Logger.getLogger(DescriptiveDataSetService.class);
  89
  90     @Autowired
  91     private IOccurrenceService occurrenceService;
  92
  93     @Autowired
  94     private ITaxonService taxonService;
  95
  96     @Autowired
  97     private IPolytomousKeyService polytomousKeyService;
  98
  99     @Autowired
 100     private IDefinedTermDao termDao;
 101
 102     @Autowired
 103     private IDescriptionService descriptionService;
 104
 105     @Autowired
 106     private ITaxonNodeService taxonNodeService;
 107
 108         @Override
 109         @Autowired
 110         protected void setDao(IDescriptiveDataSetDao dao) {
 111                 this.dao = dao;
 112         }
 113
 114         @Override
 115     public Map<DescriptionBase, Set<DescriptionElementBase>> getDescriptionElements(DescriptiveDataSet descriptiveDataSet, Set<Feature> features, Integer pageSize,     Integer pageNumber,
 116                         List<String> propertyPaths) {
 117                 return dao.getDescriptionElements(descriptiveDataSet, features, pageSize, pageNumber, propertyPaths);
 118         }
 119
 120         @Override
 121         public <T extends DescriptionElementBase> Map<UuidAndTitleCache, Map<UUID, Set<T>>> getTaxonFeatureDescriptionElementMap(
 122                         Class<T> clazz, UUID descriptiveDataSetUuid, DescriptiveSystemRole role) {
 123                 return dao.getTaxonFeatureDescriptionElementMap(clazz, descriptiveDataSetUuid, role);
 124         }
 125
 126         @Override
 127     public List<UuidAndTitleCache<DescriptiveDataSet>> getDescriptiveDataSetUuidAndTitleCache(Integer limitOfInitialElements, String pattern) {
 128         return dao.getDescriptiveDataSetUuidAndTitleCache( limitOfInitialElements, pattern);
 129     }
 130
 131         @Override
 132         public List<RowWrapperDTO<?>> getRowWrapper(UUID descriptiveDataSetUuid, IProgressMonitor monitor) {
 133             DescriptiveDataSet descriptiveDataSet = load(descriptiveDataSetUuid);
 134             monitor.beginTask("Load row wrapper", descriptiveDataSet.getDescriptions().size());
 135             List<RowWrapperDTO<?>> wrappers = new ArrayList<>();
 136             Set<DescriptionBase> descriptions = descriptiveDataSet.getDescriptions();
 137             for (DescriptionBase<?> description : descriptions) {
 138             if(monitor.isCanceled()){
 139                 return new ArrayList<>();
 140             }
 141             RowWrapperDTO<?> rowWrapper = null;
 142             // only viable descriptions are aggregated, literature or default descriptions
 143             if(HibernateProxyHelper.isInstanceOf(description, TaxonDescription.class) &&
 144                     (description.isAggregatedStructuredDescription()
 145                             || description.getTypes().contains(DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION)
 146                             || description.getTypes().contains(DescriptionType.SECONDARY_DATA)
 147                             )){
 148                 rowWrapper = createTaxonRowWrapper((TaxonDescription)description, descriptiveDataSet.getUuid());
 149             }
 150             else if (HibernateProxyHelper.isInstanceOf(description, SpecimenDescription.class)&&
 151                     !description.getTypes().contains(DescriptionType.CLONE_FOR_SOURCE)){
 152                 rowWrapper = createSpecimenRowWrapper(HibernateProxyHelper.deproxy(description, SpecimenDescription.class), descriptiveDataSetUuid);
 153             }
 154             if(rowWrapper!=null){
 155                 wrappers.add(rowWrapper);
 156             }
 157             monitor.worked(1);
 158         }
 159             return wrappers;
 160         }
 161
 162     @Override
 163     public Collection<SpecimenNodeWrapper> loadSpecimens(DescriptiveDataSet descriptiveDataSet){
 164         List<UUID> filteredNodes = findFilteredTaxonNodes(descriptiveDataSet);
 165         if(filteredNodes.isEmpty()){
 166             return Collections.emptySet();
 167         }
 168         Collection<SpecimenNodeWrapper> result = occurrenceService.listUuidAndTitleCacheByAssociatedTaxon(filteredNodes, null, null);
 169
 170         return result;
 171     }
 172
 173     @Override
 174     public Collection<SpecimenNodeWrapper> loadSpecimens(UUID descriptiveDataSetUuid){
 175         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 176         return loadSpecimens(dataSet);
 177     }
 178
 179
 180     @Override
 181     public List<UUID> findFilteredTaxonNodes(DescriptiveDataSet descriptiveDataSet){
 182         TaxonNodeFilter filter = TaxonNodeFilter.NewRankInstance(descriptiveDataSet.getMinRank(), descriptiveDataSet.getMaxRank());
 183         descriptiveDataSet.getGeoFilter().forEach(area -> filter.orArea(area.getUuid()));
 184         descriptiveDataSet.getTaxonSubtreeFilter().forEach(node -> filter.orSubtree(node));
 185         filter.setIncludeUnpublished(true);
 186
 187         return taxonNodeService.uuidList(filter);
 188     }
 189
 190     @Override
 191     public List<TaxonNode> loadFilteredTaxonNodes(DescriptiveDataSet descriptiveDataSet, List<String> propertyPaths){
 192         return taxonNodeService.load(findFilteredTaxonNodes(descriptiveDataSet), propertyPaths);
 193     }
 194
 195     @Override
 196     public TaxonDescription findDefaultDescription(UUID specimenDescriptionUuid, UUID dataSetUuid){
 197         SpecimenDescription specimenDescription = (SpecimenDescription) descriptionService.load(specimenDescriptionUuid);
 198         DescriptiveDataSet dataSet = load(dataSetUuid);
 199         TaxonNode node = findTaxonNodeForDescription(specimenDescription, dataSet);
 200         return recurseDefaultDescription(node, dataSet);
 201     }
 202
 203     private TaxonDescription recurseDefaultDescription(TaxonNode node, DescriptiveDataSet dataSet){
 204         TaxonDescription defaultDescription = null;
 205         if(node!=null && node.getTaxon()!=null){
 206             defaultDescription = findTaxonDescriptionByDescriptionType(dataSet, node.getTaxon(), DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION);
 207             if(defaultDescription==null && node.getParent()!=null){
 208                 defaultDescription = recurseDefaultDescription(node.getParent(), dataSet);
 209             }
 210         }
 211         return defaultDescription;
 212     }
 213
 214     private TaxonNode findTaxonNodeForDescription(SpecimenDescription description, DescriptiveDataSet descriptiveDataSet){
 215         SpecimenOrObservationBase<?> specimen = description.getDescribedSpecimenOrObservation();
 216         //get taxon node
 217
 218         @SuppressWarnings({ "unchecked", "cast" })
 219         //NOTE: don't remove cast as it does not compile on some systems
 220         Set<IndividualsAssociation> associations = (Set<IndividualsAssociation>)descriptiveDataSet.getDescriptions()
 221                 .stream()
 222                 .flatMap(desc->desc.getElements().stream())// put all description element in one stream
 223                 .filter(element->element instanceof IndividualsAssociation)
 224                 .map(ia->(IndividualsAssociation)ia)
 225                 .collect(Collectors.toSet());
 226         Classification classification = descriptiveDataSet.getTaxonSubtreeFilter().iterator().next().getClassification();
 227         for (IndividualsAssociation individualsAssociation : associations) {
 228             if(individualsAssociation.getAssociatedSpecimenOrObservation().equals(specimen)){
 229                 return ((TaxonDescription) individualsAssociation.getInDescription()).getTaxon().getTaxonNode(classification);
 230             }
 231         }
 232         return null;
 233     }
 234
 235     @Override
 236     public TaxonRowWrapperDTO createTaxonRowWrapper(TaxonDescription taxonDescription, UUID descriptiveDataSetUuid) {
 237         Classification classification = null;
 238         DescriptionBaseDto description = DescriptionBaseDto.fromDescription(taxonDescription);
 239
 240         DescriptiveDataSet descriptiveDataSet = dao.load(descriptiveDataSetUuid, null);
 241         Optional<TaxonNode> first = descriptiveDataSet.getTaxonSubtreeFilter().stream()
 242                 .filter(node->node.getClassification()!=null).findFirst();
 243         Optional<Classification> classificationOptional = first.map(node->node.getClassification());
 244         Set<DescriptionBaseDto> descriptions = new HashSet<>();
 245         TaxonNodeDto nodeDto = null;
 246         if(classificationOptional.isPresent()){
 247             classification = classificationOptional.get();
 248             nodeDto = taxonNodeService.dto(description.getTaxonDto().getUuid(), classification.getUuid());
 249         }
 250
 251         return new TaxonRowWrapperDTO(description, nodeDto, descriptions);
 252     }
 253
 254     @Override
 255     @Transactional(readOnly=false)
 256     public UpdateResult addRowWrapperToDataset(Collection<SpecimenRowWrapperDTO> wrappers, UUID datasetUuid){
 257         UpdateResult result = new UpdateResult();
 258         DescriptiveDataSet dataSet = load(datasetUuid);
 259         result.setCdmEntity(dataSet);
 260
 261         List<UUID> taxonUuids = wrappers.stream().map(wrapper->wrapper.getTaxonNode().getTaxonUuid()).collect(Collectors.toList());
 262         List<TaxonBase> taxa = taxonService.load(taxonUuids, Arrays.asList(new String[]{"descriptions"}));
 263
 264         for (SpecimenRowWrapperDTO wrapper : wrappers) {
 265             Optional<TaxonBase> findAny = taxa.stream().filter(taxon->taxon.getUuid().equals(wrapper.getTaxonNode().getTaxonUuid())).findAny();
 266             if(!findAny.isPresent()){
 267                 result.addException(new IllegalArgumentException("Could not create wrapper for "+ wrapper.getSpecimenDto().getLabel()));
 268                 continue;
 269             }
 270             Taxon taxon = (Taxon) findAny.get();
 271
 272             SpecimenOrObservationBase<?> specimen = occurrenceService.load(wrapper.getSpecimenDto().getUuid());
 273
 274             TaxonDescription taxonDescription = taxon.getDescriptions().stream()
 275                     .filter(desc->desc.getTypes().contains(DescriptionType.INDIVIDUALS_ASSOCIATION))
 276                     .findFirst().orElseGet(()->{
 277                         TaxonDescription td = TaxonDescription.NewInstance(taxon);
 278                         td.addType(DescriptionType.INDIVIDUALS_ASSOCIATION);
 279                         td.setTitleCache("Specimens used by " + dataSet.getTitleCache() + " for " + taxon.getTitleCache(), true);
 280                         return td;});
 281
 282             IndividualsAssociation association = IndividualsAssociation.NewInstance(specimen);
 283             taxonDescription.addElement(association);
 284             taxonService.saveOrUpdate(taxon);
 285             result.addUpdatedObject(taxon);
 286
 287             UUID specimenDescriptionUuid = wrapper.getDescription().getDescriptionUuid();
 288             DescriptionBaseDto descriptionDto = wrapper.getDescription();
 289             DescriptionBase<?> specimenDescription =  descriptionService.load(specimenDescriptionUuid);
 290             //if description already exist use the loaded one and add changed data otherwise create a new one and add to specimen
 291             if (specimenDescription == null){
 292                 specimenDescription = SpecimenDescription.NewInstance(specimen);
 293                 specimenDescription.setUuid(specimenDescriptionUuid);
 294                 List<DescriptionElementDto> elementDtos = descriptionDto.getElements();
 295
 296                 for (DescriptionElementDto elementDto: elementDtos){
 297                     if (elementDto instanceof CategoricalDataDto){
 298                         eu.etaxonomy.cdm.model.description.Character feature = DefinedTermBase.getTermByClassAndUUID(eu.etaxonomy.cdm.model.description.Character.class, elementDto.getFeatureUuid());
 299                         CategoricalData data = CategoricalData.NewInstance(feature);
 300                         for (StateDataDto stateDto:((CategoricalDataDto) elementDto).getStates()){
 301                             State state = DefinedTermBase.getTermByClassAndUUID(State.class, stateDto.getState().getUuid());
 302                             data.addStateData(state);
 303                             specimenDescription.addElement(data);
 304                         }
 305                     }
 306                     if (elementDto instanceof QuantitativeDataDto){
 307                         eu.etaxonomy.cdm.model.description.Character feature = DefinedTermBase.getTermByClassAndUUID(eu.etaxonomy.cdm.model.description.Character.class, elementDto.getFeatureUuid());
 308                         QuantitativeData data = QuantitativeData.NewInstance(feature);
 309                         if (((QuantitativeDataDto) elementDto).getMeasurementUnit() != null){
 310                             MeasurementUnit unit = DefinedTermBase.getTermByClassAndUUID(MeasurementUnit.class, ((QuantitativeDataDto) elementDto).getMeasurementUnit().getUuid());
 311                             data.setUnit(unit);
 312                         }
 313
 314                         for (StatisticalMeasurementValueDto stateDto:((QuantitativeDataDto) elementDto).getValues()){
 315                             StatisticalMeasure statMeasure = DefinedTermBase.getTermByClassAndUUID(StatisticalMeasure.class, stateDto.getType().getUuid());
 316                             StatisticalMeasurementValue value = StatisticalMeasurementValue.NewInstance(statMeasure, stateDto.getValue());
 317                             data.addStatisticalValue(value);
 318                             specimenDescription.addElement(data);
 319                         }
 320                     }
 321                 }
 322
 323             }else {
 324                 List<DescriptionElementDto> elementDtos = descriptionDto.getElements();
 325                 for (DescriptionElementDto elementDto: elementDtos){
 326                     if (elementDto instanceof CategoricalDataDto){
 327                         eu.etaxonomy.cdm.model.description.Character feature = DefinedTermBase.getTermByClassAndUUID(eu.etaxonomy.cdm.model.description.Character.class, elementDto.getFeatureUuid());
 328                         List<DescriptionElementBase> uniqueElementList = specimenDescription.getElements().stream().filter(element -> element.getUuid().equals(elementDto.getElementUuid())).collect(Collectors.toList());
 329                         List<State> allStates = new ArrayList<>();
 330                         CategoricalData element = null;
 331                         if (uniqueElementList.size() == 1){
 332                             element = HibernateProxyHelper.deproxy(uniqueElementList.get(0), CategoricalData.class);
 333                         }else{
 334                             element = CategoricalData.NewInstance(feature);
 335                         }
 336                         for (StateDataDto stateDto:((CategoricalDataDto) elementDto).getStates()){
 337                             State state = DefinedTermBase.getTermByClassAndUUID(State.class, stateDto.getState().getUuid());
 338                             allStates.add(state);
 339                         }
 340                         element.setStateDataOnly(allStates);
 341                     }
 342                     if (elementDto instanceof QuantitativeDataDto){
 343                         eu.etaxonomy.cdm.model.description.Character feature = DefinedTermBase.getTermByClassAndUUID(eu.etaxonomy.cdm.model.description.Character.class, elementDto.getFeatureUuid());
 344                         QuantitativeData data = QuantitativeData.NewInstance(feature);
 345                         if (((QuantitativeDataDto) elementDto).getMeasurementUnit() != null){
 346                             MeasurementUnit unit = DefinedTermBase.getTermByClassAndUUID(MeasurementUnit.class, ((QuantitativeDataDto) elementDto).getMeasurementUnit().getUuid());
 347                             data.setUnit(unit);
 348                         }
 349
 350                         for (StatisticalMeasurementValueDto stateDto:((QuantitativeDataDto) elementDto).getValues()){
 351                             StatisticalMeasure statMeasure = DefinedTermBase.getTermByClassAndUUID(StatisticalMeasure.class, stateDto.getType().getUuid());
 352                             StatisticalMeasurementValue value = StatisticalMeasurementValue.NewInstance(statMeasure, stateDto.getValue());
 353                             data.addStatisticalValue(value);
 354                             specimenDescription.addElement(data);
 355                         }
 356                     }
 357                 }
 358             }
 359
 360             //add specimen description to data set
 361             specimenDescription.addDescriptiveDataSet(dataSet);
 362             //add taxon description with IndividualsAssociation to the specimen to data set
 363             taxonDescription.addDescriptiveDataSet(dataSet);
 364             result.addUpdatedObject(specimen);
 365             result.addUpdatedObject(specimenDescription);
 366             result.addUpdatedObject(taxonDescription);
 367         }
 368         saveOrUpdate(dataSet);
 369         return result;
 370     }
 371
 372     private SpecimenRowWrapperDTO createSpecimenRowWrapper(SpecimenDescription description, UUID taxonNodeUuid,
 373             UUID datasetUuid) {
 374         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid);
 375         DescriptiveDataSet descriptiveDataSet = load(datasetUuid);
 376         SpecimenOrObservationBase<?> specimen = HibernateProxyHelper.deproxy(description.getDescribedSpecimenOrObservation(), SpecimenOrObservationBase.class);
 377         //supplemental information
 378         if(taxonNode==null){
 379             taxonNode = findTaxonNodeForDescription(description, descriptiveDataSet);
 380         }
 381         FieldUnit fieldUnit = null;
 382         String identifier = null;
 383         NamedArea country = null;
 384         if(taxonNode==null){
 385             return null;
 386         }
 387         //taxon node was found
 388
 389         //get field unit
 390         Collection<FieldUnit> fieldUnits = occurrenceService.findFieldUnits(specimen.getUuid(),
 391                 Arrays.asList(new String[]{
 392                         "gatheringEvent",
 393                         "gatheringEvent.country"
 394                 }));
 395         if(fieldUnits.size()>1){
 396             logger.error("More than one or no field unit found for specimen"); //$NON-NLS-1$
 397             return null;
 398         }
 399         else{
 400             if (fieldUnits.size()>0){
 401                 fieldUnit = fieldUnits.iterator().next();
 402             }
 403         }
 404         //get identifier
 405         if(HibernateProxyHelper.isInstanceOf(specimen, DerivedUnit.class)){
 406             identifier = occurrenceService.getMostSignificantIdentifier(HibernateProxyHelper.deproxy(specimen, DerivedUnit.class));
 407         }
 408         //get country
 409         if(fieldUnit != null && fieldUnit.getGatheringEvent() != null){
 410             country = fieldUnit.getGatheringEvent().getCountry();
 411         }
 412         //get default taxon description
 413 //        TaxonDescription defaultTaxonDescription = findDefaultDescription(description.getUuid(), descriptiveDataSet.getUuid());
 414         TaxonDescription defaultTaxonDescription = recurseDefaultDescription(taxonNode, descriptiveDataSet);
 415         TaxonRowWrapperDTO taxonRowWrapper = defaultTaxonDescription != null
 416                 ? createTaxonRowWrapper(defaultTaxonDescription, descriptiveDataSet.getUuid()) : null;
 417 //                use description not specimen for specimenRow
 418         SpecimenRowWrapperDTO specimenRowWrapperDTO = new SpecimenRowWrapperDTO(DescriptionBaseDto.fromDescription(description), SpecimenOrObservationDTOFactory.fromEntity(specimen), specimen.getRecordBasis(), new TaxonNodeDto(taxonNode), fieldUnit, identifier, country);
 419         specimenRowWrapperDTO.setDefaultDescription(taxonRowWrapper);
 420         return specimenRowWrapperDTO;
 421     }
 422
 423     @Override
 424     public SpecimenRowWrapperDTO createSpecimenRowWrapper(SpecimenDescription description, UUID descriptiveDataSetUuid){
 425         return createSpecimenRowWrapper(description, null, descriptiveDataSetUuid);
 426         }
 427
 428     @Override
 429     public SpecimenRowWrapperDTO createSpecimenRowWrapper(UUID specimenUuid, UUID taxonNodeUuid, UUID descriptiveDataSetUuid){
 430
 431         SpecimenOrObservationBase<?> specimen = occurrenceService.load(specimenUuid);
 432         SpecimenDescription specimenDescription = findSpecimenDescription(descriptiveDataSetUuid, specimen, true);
 433         return createSpecimenRowWrapper(specimenDescription, taxonNodeUuid, descriptiveDataSetUuid);
 434     }
 435
 436     @Override
 437     @Transactional(readOnly = false)
 438     public UpdateResult updateCaches(Class<? extends DescriptiveDataSet> clazz, Integer stepSize,
 439             IIdentifiableEntityCacheStrategy<DescriptiveDataSet> cacheStrategy, IProgressMonitor monitor) {
 440         if (clazz == null) {
 441             clazz = DescriptiveDataSet.class;
 442         }
 443         return super.updateCachesImpl(clazz, stepSize, cacheStrategy, monitor);
 444     }
 445
 446     private TaxonDescription findTaxonDescriptionByDescriptionType(DescriptiveDataSet dataSet, Taxon taxon, DescriptionType descriptionType){
 447         Optional<TaxonDescription> first = taxon.getDescriptions().stream()
 448                 .filter(desc -> desc.getTypes().stream().anyMatch(type -> type.equals(descriptionType)))
 449                 .filter(desc -> dataSet.getDescriptions().contains(desc))
 450                 .findFirst();
 451         if(first.isPresent()){
 452             return HibernateProxyHelper.deproxy(descriptionService.load(first.get().getUuid(),
 453                   Arrays.asList("taxon", "descriptionElements", "descriptionElements.feature")), TaxonDescription.class);
 454         }
 455         return null;
 456     }
 457
 458     @Override
 459     public TaxonDescription findTaxonDescriptionByDescriptionType(UUID dataSetUuid, UUID taxonNodeUuid, DescriptionType descriptionType){
 460         DescriptiveDataSet dataSet = load(dataSetUuid);
 461         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid);
 462         return findTaxonDescriptionByDescriptionType(dataSet, taxonNode.getTaxon(), descriptionType);
 463     }
 464
 465     @Override
 466     @Transactional(readOnly=false)
 467     public UpdateResult generatePolytomousKey(UUID descriptiveDataSetUuid, UUID taxonUuid) {
 468         UpdateResult result = new UpdateResult();
 469
 470         PolytomousKeyGeneratorConfigurator keyConfig = new PolytomousKeyGeneratorConfigurator();
 471         DescriptiveDataSet descriptiveDataSet = load(descriptiveDataSetUuid);
 472         keyConfig.setDataSet(descriptiveDataSet);
 473         PolytomousKey key = new PolytomousKeyGenerator().invoke(keyConfig);
 474         IdentifiableServiceConfiguratorImpl<PolytomousKey> serviceConfig= new IdentifiableServiceConfiguratorImpl<>();
 475         serviceConfig.setTitleSearchString(descriptiveDataSet.getTitleCache());
 476         List<PolytomousKey> list = polytomousKeyService.findByTitle(serviceConfig).getRecords();
 477         if(list!=null){
 478             list.forEach(polytomousKey->polytomousKeyService.delete(polytomousKey));
 479         }
 480         key.setTitleCache(descriptiveDataSet.getTitleCache(), true);
 481
 482         Taxon taxon = (Taxon) taxonService.load(taxonUuid);
 483         key.addTaxonomicScope(taxon);
 484
 485         polytomousKeyService.saveOrUpdate(key);
 486
 487         result.setCdmEntity(key);
 488         result.addUpdatedObject(taxon);
 489         return result;
 490     }
 491
 492     @Override
 493     @Transactional(readOnly=false)
 494     public DeleteResult removeDescription(UUID descriptionUuid, UUID descriptiveDataSetUuid, RemoveDescriptionsFromDescriptiveDataSetConfigurator config) {
 495         DeleteResult result = new DeleteResult();
 496         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 497         DescriptionBase<?> descriptionBase = descriptionService.load(descriptionUuid);
 498         if(dataSet==null || descriptionBase==null){
 499             result.setError();
 500         }
 501         else{
 502             removeDescriptionFromDataSet(result, dataSet, descriptionBase, config);
 503         }
 504         return result;
 505     }
 506
 507
 508     @Override
 509     @Transactional(readOnly=false)
 510     public DeleteResult removeDescriptions(List<UUID> descriptionUuids, UUID descriptiveDataSetUuid, RemoveDescriptionsFromDescriptiveDataSetConfigurator config) {
 511         DeleteResult result = new DeleteResult();
 512         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 513         List<DescriptionBase> descriptions = descriptionService.load(descriptionUuids, null);
 514         if(dataSet==null || descriptions==null){
 515             result.setError();
 516         }
 517         else{
 518             for (DescriptionBase<?> description: descriptions){
 519                 removeDescriptionFromDataSet(result, dataSet, description, config);
 520             }
 521         }
 522         return result;
 523     }
 524
 525     private void removeDescriptionFromDataSet(DeleteResult result, DescriptiveDataSet dataSet,
 526             DescriptionBase<?> description, RemoveDescriptionsFromDescriptiveDataSetConfigurator config) {
 527         if (description == null){
 528             return;
 529         }
 530         boolean success = dataSet.removeDescription(description);
 531         result.addDeletedObject(description);// remove taxon description with IndividualsAssociation from data set
 532         if(description instanceof SpecimenDescription){
 533             @SuppressWarnings({ "unchecked", "cast" })
 534             //NOTE: don't remove cast as it does not compile on some systems
 535             Set<IndividualsAssociation> associations = (Set<IndividualsAssociation>)dataSet.getDescriptions()
 536                     .stream()
 537                     .flatMap(desc->desc.getElements().stream())// put all description element in one stream
 538                     .filter(element->element instanceof IndividualsAssociation)
 539                     .map(ia->(IndividualsAssociation)ia)
 540                     .collect(Collectors.toSet());
 541
 542             for (IndividualsAssociation individualsAssociation : associations) {
 543                 if(individualsAssociation.getAssociatedSpecimenOrObservation().equals(description.getDescribedSpecimenOrObservation())){
 544                     dataSet.removeDescription(individualsAssociation.getInDescription());
 545                     result.addUpdatedObject(individualsAssociation.getInDescription());
 546                 }
 547             }
 548         }
 549         if (description instanceof TaxonDescription){
 550             DeleteResult isDeletable = descriptionService.isDeletable(description.getUuid());
 551             for (CdmBase relatedCdmBase: isDeletable.getRelatedObjects()){
 552                 if (relatedCdmBase instanceof CdmLinkSource){
 553                     CdmLinkSource linkSource = (CdmLinkSource)relatedCdmBase;
 554                     if (linkSource.getTarget().equals(this)){
 555
 556                     }
 557                 }
 558             }
 559
 560
 561         }
 562         if (!config.isOnlyRemoveDescriptionsFromDataSet()){
 563             DeleteResult deleteResult = descriptionService.deleteDescription(description);
 564             result.includeResult(deleteResult);
 565             result.addUpdatedObject(dataSet);
 566         }else{
 567             MergeResult<DescriptiveDataSet> mergeResult = dao.merge(dataSet, true);
 568             result.addUpdatedObject(mergeResult.getMergedEntity());
 569         }
 570
 571         result.setStatus(success?Status.OK:Status.ERROR);
 572     }
 573
 574     @Override
 575     @Transactional(readOnly = false)
 576     public DeleteResult delete(UUID datasetUuid, DeleteDescriptiveDataSetConfigurator config,  IProgressMonitor monitor){
 577         DescriptiveDataSet dataSet = dao.load(datasetUuid);
 578         monitor.beginTask("Delete Descriptive Dataset", dataSet.getDescriptions().size() +1);
 579
 580         DeleteResult result = new DeleteResult();
 581         DeleteResult descriptionResult = new DeleteResult();
 582         if (!dataSet.getDescriptions().isEmpty()){
 583             Set<DescriptionBase> descriptions = new HashSet();;
 584             for (DescriptionBase desc: dataSet.getDescriptions()){
 585                 descriptions.add(desc);
 586             }
 587             monitor.subTask("Delete descriptions");
 588             for (DescriptionBase desc: descriptions){
 589                 dataSet.removeDescription(desc);
 590                 if (desc instanceof SpecimenDescription && config.isDeleteAllSpecimenDescriptions()){
 591                     descriptionResult.includeResult(descriptionService.deleteDescription(desc));
 592                 }else if (desc instanceof TaxonDescription){
 593                     if( desc.getTypes().contains(DescriptionType.DEFAULT_VALUES_FOR_AGGREGATION) && config.isDeleteAllDefaultDescriptions()){
 594                         descriptionResult.includeResult(descriptionService.deleteDescription(desc));
 595                     }else if (desc.getTypes().contains(DescriptionType.SECONDARY_DATA) && config.isDeleteAllLiteratureDescriptions()){
 596                         descriptionResult.includeResult(descriptionService.deleteDescription(desc));
 597                     }else if (desc.getTypes().contains(DescriptionType.AGGREGATED_STRUC_DESC) && config.isDeleteAllAggregatedDescriptions()){
 598                         descriptionResult.includeResult(descriptionService.deleteDescription(desc));
 599                     }
 600
 601                 }
 602
 603             }
 604
 605
 606         }
 607         dao.delete(dataSet);
 608         monitor.worked(1);
 609         monitor.done();
 610         result.includeResult(descriptionResult);
 611         result.setStatus(Status.OK);
 612         result.addDeletedObject(dataSet);
 613         return result;
 614     }
 615
 616     @Override
 617     @Transactional(readOnly=false)
 618     public TaxonRowWrapperDTO createTaxonDescription(UUID dataSetUuid, UUID taxonNodeUuid, DescriptionType descriptionType){
 619         DescriptiveDataSet dataSet = load(dataSetUuid);
 620         TaxonNode taxonNode = taxonNodeService.load(taxonNodeUuid, Arrays.asList("taxon"));
 621         TaxonDescription newTaxonDescription = TaxonDescription.NewInstance(taxonNode.getTaxon());
 622         newTaxonDescription.setTitleCache(dataSet.getLabel()+": "+newTaxonDescription.generateTitle(), true); //$NON-NLS-2$
 623         newTaxonDescription.getTypes().add(descriptionType);
 624         dataSet.addDescription(newTaxonDescription);
 625         saveOrUpdate(dataSet);
 626         return createTaxonRowWrapper(newTaxonDescription, dataSet.getUuid());
 627     }
 628
 629     @Override
 630     public Map<UUID, List<TermDto>> getSupportedStatesForFeature(Set<UUID> featureUuids){
 631         return termDao.getSupportedStatesForFeature(featureUuids);
 632     }
 633
 634     @Override
 635     @Transactional(readOnly=false)
 636     public SpecimenDescription findSpecimenDescription(UUID descriptiveDataSetUuid, SpecimenOrObservationBase specimen, boolean addDatasetSource){
 637         DescriptiveDataSet dataSet = load(descriptiveDataSetUuid);
 638 //        SpecimenOrObservationBase specimen = occurrenceService.load(specimenUuid);
 639
 640         Set<? extends Feature> datasetFeatures = dataSet.getDescriptiveSystem().getDistinctTerms();
 641         List<DescriptionElementBase> matchingDescriptionElements = new ArrayList<>();
 642
 643         for (SpecimenDescription specimenDescription : (Set<SpecimenDescription>) specimen.getDescriptions()) {
 644             specimenDescription = (SpecimenDescription) descriptionService.load(specimenDescription.getUuid());
 645
 646             //check if description is already added to data set
 647             if(dataSet.getDescriptions().contains(specimenDescription) ){
 648                 return specimenDescription;
 649             }
 650
 651             //gather specimen description features and check for match with dataset features
 652             Set<Feature> specimenDescriptionFeatures = new HashSet<>();
 653             for (DescriptionElementBase specimenDescriptionElement : specimenDescription.getElements()) {
 654                 Feature feature = specimenDescriptionElement.getFeature();
 655                 specimenDescriptionFeatures.add(feature);
 656                 if(datasetFeatures.contains(feature) && RowWrapperDTO.hasData(specimenDescriptionElement)){
 657                     matchingDescriptionElements.add(specimenDescriptionElement);
 658                 }
 659             }
 660         }
 661         //Create new specimen description if description has not already been added to the dataset
 662         SpecimenDescription newDesription = SpecimenDescription.NewInstance(specimen);
 663         newDesription.setTitleCache("Dataset "+dataSet.getLabel()+": "+newDesription.generateTitle(), true); //$NON-NLS-2$
 664
 665         //check for equals description element (same feature and same values)
 666         Map<Feature, List<DescriptionElementBase>> featureToElementMap = new HashMap<>();
 667         for(DescriptionElementBase element:matchingDescriptionElements){
 668             List<DescriptionElementBase> list = featureToElementMap.get(element.getFeature());
 669             if(list==null){
 670                 list = new ArrayList<>();
 671             }
 672             list.add(element);
 673             featureToElementMap.put(element.getFeature(), list);
 674         }
 675         Set<DescriptionElementBase> descriptionElementsToClone = new HashSet<>();
 676         for(Feature feature:featureToElementMap.keySet()){
 677             List<DescriptionElementBase> elements = featureToElementMap.get(feature);
 678             //no duplicate description elements found for this feature
 679             if(elements.size()==1){
 680                 descriptionElementsToClone.add(elements.get(0));
 681             }
 682             //duplicates found -> check if all are equal
 683             else{
 684                 DescriptionElementBase match = null;
 685                 for (DescriptionElementBase descriptionElementBase : elements) {
 686                     if(match==null){
 687                         match = descriptionElementBase;
 688                     }
 689                     else if(!new DescriptionElementCompareWrapper(match).equals(new DescriptionElementCompareWrapper(descriptionElementBase))){
 690                         match = null;
 691                         //TODO: propagate message
 692 //                        MessagingUtils.informationDialog(Messages.CharacterMatrix_MULTIPLE_DATA,
 693 //                                String.format(Messages.CharacterMatrix_MULTIPLE_DATA_MESSAGE, feature.getLabel()));
 694                         break;
 695                     }
 696                 }
 697                 if(match!=null){
 698                     descriptionElementsToClone.add(match);
 699                 }
 700             }
 701         }
 702         //clone matching descriptionElements
 703         for (DescriptionElementBase descriptionElementBase : descriptionElementsToClone) {
 704             DescriptionElementBase clone;
 705             clone = descriptionElementBase.clone(newDesription);
 706             clone.getSources().forEach(source -> {
 707                 if(descriptionElementBase instanceof CategoricalData){
 708                     TextData label = new DefaultCategoricalDescriptionBuilder().build((CategoricalData) descriptionElementBase, Arrays.asList(new Language[]{Language.DEFAULT()}));
 709                     source.setOriginalNameString(label.getText(Language.DEFAULT()));
 710                 }
 711                 else if(descriptionElementBase instanceof QuantitativeData){
 712                     TextData label = new DefaultQuantitativeDescriptionBuilder().build((QuantitativeData) descriptionElementBase, Arrays.asList(new Language[]{Language.DEFAULT()}));
 713                     source.setOriginalNameString(label.getText(Language.DEFAULT()));
 714                 }
 715             });
 716         }
 717
 718         //add sources of data set
 719         if(addDatasetSource){
 720             dataSet.getSources().forEach(source->{
 721                 try {
 722                     newDesription.addSource(source.clone());
 723                 } catch (CloneNotSupportedException e) {
 724                     //nothing
 725                 }
 726             });
 727         }
 728         return newDesription;
 729
 730     }
 731
 732     //TODO: this should either be solved in the model class itself
 733     //OR this should cover all possibilities including modifiers for example
 734     private class DescriptionElementCompareWrapper {
 735
 736         private DescriptionElementBase element;
 737         private Set<UUID> stateUuids = new HashSet<>();
 738         private Set<BigDecimal> avgs = new HashSet<>();
 739         private Set<BigDecimal> exacts = new HashSet<>();
 740         private Set<BigDecimal> maxs = new HashSet<>();
 741         private Set<BigDecimal> mins = new HashSet<>();
 742         private Set<BigDecimal> sampleSizes = new HashSet<>();
 743         private Set<BigDecimal> standardDevs = new HashSet<>();
 744         private Set<BigDecimal> lowerBounds = new HashSet<>();
 745         private Set<BigDecimal> upperBounds = new HashSet<>();
 746         private Set<BigDecimal> variances = new HashSet<>();
 747
 748         public DescriptionElementCompareWrapper(DescriptionElementBase element) {
 749             this.element = element;
 750             if(element.isInstanceOf(CategoricalData.class)){
 751                 CategoricalData elementData = (CategoricalData)element;
 752                 elementData.getStatesOnly().forEach(state->stateUuids.add(state.getUuid()));
 753             }
 754             else if(element.isInstanceOf(QuantitativeData.class)){
 755                 QuantitativeData elementData = (QuantitativeData)element;
 756                 elementData.getStatisticalValues().forEach(value->{
 757                     if(value.getType().equals(StatisticalMeasure.AVERAGE())){
 758                         avgs.add(value.getValue());
 759                     }
 760                     else if(value.getType().equals(StatisticalMeasure.EXACT_VALUE())){
 761                         exacts.add(value.getValue());
 762
 763                     }
 764                     else if(value.getType().equals(StatisticalMeasure.MAX())){
 765                         maxs.add(value.getValue());
 766                     }
 767                     else if(value.getType().equals(StatisticalMeasure.MIN())){
 768                         mins.add(value.getValue());
 769                     }
 770                     else if(value.getType().equals(StatisticalMeasure.SAMPLE_SIZE())){
 771                         sampleSizes.add(value.getValue());
 772
 773                     }
 774                     else if(value.getType().equals(StatisticalMeasure.STANDARD_DEVIATION())){
 775                         standardDevs.add(value.getValue());
 776                     }
 777                     else if(value.getType().equals(StatisticalMeasure.TYPICAL_LOWER_BOUNDARY())){
 778                         lowerBounds.add(value.getValue());
 779
 780                     }
 781                     else if(value.getType().equals(StatisticalMeasure.TYPICAL_UPPER_BOUNDARY())){
 782                         upperBounds.add(value.getValue());
 783                     }
 784                     else if(value.getType().equals(StatisticalMeasure.VARIANCE())){
 785                         variances.add(value.getValue());
 786                     }
 787                 });
 788             }
 789         }
 790
 791         @Override
 792         public int hashCode() {
 793             final int prime = 31;
 794             int result = 1;
 795             result = prime * result + getOuterType().hashCode();
 796             result = prime * result + ((avgs == null) ? 0 : avgs.hashCode());
 797             result = prime * result + ((element == null) ? 0 : element.hashCode());
 798             result = prime * result + ((exacts == null) ? 0 : exacts.hashCode());
 799             result = prime * result + ((lowerBounds == null) ? 0 : lowerBounds.hashCode());
 800             result = prime * result + ((maxs == null) ? 0 : maxs.hashCode());
 801             result = prime * result + ((mins == null) ? 0 : mins.hashCode());
 802             result = prime * result + ((sampleSizes == null) ? 0 : sampleSizes.hashCode());
 803             result = prime * result + ((standardDevs == null) ? 0 : standardDevs.hashCode());
 804             result = prime * result + ((stateUuids == null) ? 0 : stateUuids.hashCode());
 805             result = prime * result + ((upperBounds == null) ? 0 : upperBounds.hashCode());
 806             result = prime * result + ((variances == null) ? 0 : variances.hashCode());
 807             return result;
 808         }
 809
 810         @Override
 811         public boolean equals(Object obj) {
 812             if (this == obj) {
 813                 return true;
 814             }
 815             if (obj == null) {
 816                 return false;
 817             }
 818             if (getClass() != obj.getClass()) {
 819                 return false;
 820             }
 821             DescriptionElementCompareWrapper other = (DescriptionElementCompareWrapper) obj;
 822             if (!getOuterType().equals(other.getOuterType())) {
 823                 return false;
 824             }
 825             if (avgs == null) {
 826                 if (other.avgs != null) {
 827                     return false;
 828                 }
 829             } else if (!avgs.equals(other.avgs)) {
 830                 return false;
 831             }
 832             if (element == null) {
 833                 if (other.element != null) {
 834                     return false;
 835                 }
 836             } else if (!element.equals(other.element)) {
 837                 return false;
 838             }
 839             if (exacts == null) {
 840                 if (other.exacts != null) {
 841                     return false;
 842                 }
 843             } else if (!exacts.equals(other.exacts)) {
 844                 return false;
 845             }
 846             if (lowerBounds == null) {
 847                 if (other.lowerBounds != null) {
 848                     return false;
 849                 }
 850             } else if (!lowerBounds.equals(other.lowerBounds)) {
 851                 return false;
 852             }
 853             if (maxs == null) {
 854                 if (other.maxs != null) {
 855                     return false;
 856                 }
 857             } else if (!maxs.equals(other.maxs)) {
 858                 return false;
 859             }
 860             if (mins == null) {
 861                 if (other.mins != null) {
 862                     return false;
 863                 }
 864             } else if (!mins.equals(other.mins)) {
 865                 return false;
 866             }
 867             if (sampleSizes == null) {
 868                 if (other.sampleSizes != null) {
 869                     return false;
 870                 }
 871             } else if (!sampleSizes.equals(other.sampleSizes)) {
 872                 return false;
 873             }
 874             if (standardDevs == null) {
 875                 if (other.standardDevs != null) {
 876                     return false;
 877                 }
 878             } else if (!standardDevs.equals(other.standardDevs)) {
 879                 return false;
 880             }
 881             if (stateUuids == null) {
 882                 if (other.stateUuids != null) {
 883                     return false;
 884                 }
 885             } else if (!stateUuids.equals(other.stateUuids)) {
 886                 return false;
 887             }
 888             if (upperBounds == null) {
 889                 if (other.upperBounds != null) {
 890                     return false;
 891                 }
 892             } else if (!upperBounds.equals(other.upperBounds)) {
 893                 return false;
 894             }
 895             if (variances == null) {
 896                 if (other.variances != null) {
 897                     return false;
 898                 }
 899             } else if (!variances.equals(other.variances)) {
 900                 return false;
 901             }
 902             return true;
 903         }
 904
 905         private DescriptiveDataSetService getOuterType() {
 906             return DescriptiveDataSetService.this;
 907         }
 908     }
 909
 910     @Override
 911     public DescriptiveDataSetBaseDto getDescriptiveDataSetDtoByUuid(UUID uuid) {
 912         return dao.getDescriptiveDataSetDtoByUuid(uuid);
 913     }
 914
 915 }