cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/TaxonServiceImpl.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import java.io.IOException;
  13 import java.util.ArrayList;
  14 import java.util.EnumSet;
  15 import java.util.HashMap;
  16 import java.util.HashSet;
  17 import java.util.Iterator;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import javax.persistence.EntityNotFoundException;
  24
  25 import org.apache.commons.lang3.StringUtils;
  26 import org.apache.log4j.Logger;
  27 import org.apache.lucene.queryparser.classic.ParseException;
  28 import org.apache.lucene.search.BooleanClause.Occur;
  29 import org.apache.lucene.search.BooleanQuery;
  30 import org.apache.lucene.search.BooleanQuery.Builder;
  31 import org.apache.lucene.search.Query;
  32 import org.apache.lucene.search.SortField;
  33 import org.apache.lucene.search.grouping.TopGroups;
  34 import org.apache.lucene.search.join.ScoreMode;
  35 import org.apache.lucene.util.BytesRef;
  36 import org.springframework.beans.factory.annotation.Autowired;
  37 import org.springframework.stereotype.Service;
  38 import org.springframework.transaction.annotation.Transactional;
  39
  40 import eu.etaxonomy.cdm.api.service.config.DeleteConfiguratorBase;
  41 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  42 import eu.etaxonomy.cdm.api.service.config.IncludedTaxonConfiguration;
  43 import eu.etaxonomy.cdm.api.service.config.MatchingTaxonConfigurator;
  44 import eu.etaxonomy.cdm.api.service.config.NodeDeletionConfigurator.ChildHandling;
  45 import eu.etaxonomy.cdm.api.service.config.SynonymDeletionConfigurator;
  46 import eu.etaxonomy.cdm.api.service.config.TaxonDeletionConfigurator;
  47 import eu.etaxonomy.cdm.api.service.dto.IdentifiedEntityDTO;
  48 import eu.etaxonomy.cdm.api.service.dto.IncludedTaxaDTO;
  49 import eu.etaxonomy.cdm.api.service.dto.MarkedEntityDTO;
  50 import eu.etaxonomy.cdm.api.service.dto.TaxonRelationshipsDTO;
  51 import eu.etaxonomy.cdm.api.service.exception.DataChangeNoRollbackException;
  52 import eu.etaxonomy.cdm.api.service.exception.HomotypicalGroupChangeException;
  53 import eu.etaxonomy.cdm.api.service.exception.ReferencedObjectUndeletableException;
  54 import eu.etaxonomy.cdm.api.service.pager.Pager;
  55 import eu.etaxonomy.cdm.api.service.pager.impl.DefaultPagerImpl;
  56 import eu.etaxonomy.cdm.api.service.search.ILuceneIndexToolProvider;
  57 import eu.etaxonomy.cdm.api.service.search.ISearchResultBuilder;
  58 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearch;
  59 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  60 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  61 import eu.etaxonomy.cdm.api.service.search.LuceneSearch;
  62 import eu.etaxonomy.cdm.api.service.search.QueryFactory;
  63 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  64 import eu.etaxonomy.cdm.api.service.search.SearchResultBuilder;
  65 import eu.etaxonomy.cdm.api.service.util.TaxonRelationshipEdge;
  66 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  67 import eu.etaxonomy.cdm.exception.UnpublishedException;
  68 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  69 import eu.etaxonomy.cdm.hibernate.search.AcceptedTaxonBridge;
  70 import eu.etaxonomy.cdm.hibernate.search.GroupByTaxonClassBridge;
  71 import eu.etaxonomy.cdm.model.CdmBaseType;
  72 import eu.etaxonomy.cdm.model.common.Annotation;
  73 import eu.etaxonomy.cdm.model.common.AnnotationType;
  74 import eu.etaxonomy.cdm.model.common.CdmBase;
  75 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  76 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  77 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  78 import eu.etaxonomy.cdm.model.common.Language;
  79 import eu.etaxonomy.cdm.model.common.MarkerType;
  80 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  81 import eu.etaxonomy.cdm.model.common.RelationshipBase.Direction;
  82 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  83 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  84 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  85 import eu.etaxonomy.cdm.model.description.Distribution;
  86 import eu.etaxonomy.cdm.model.description.Feature;
  87 import eu.etaxonomy.cdm.model.description.IIdentificationKey;
  88 import eu.etaxonomy.cdm.model.description.PolytomousKeyNode;
  89 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  90 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  91 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  92 import eu.etaxonomy.cdm.model.description.TaxonInteraction;
  93 import eu.etaxonomy.cdm.model.description.TaxonNameDescription;
  94 import eu.etaxonomy.cdm.model.location.NamedArea;
  95 import eu.etaxonomy.cdm.model.media.Media;
  96 import eu.etaxonomy.cdm.model.name.HomotypicalGroup;
  97 import eu.etaxonomy.cdm.model.name.IZoologicalName;
  98 import eu.etaxonomy.cdm.model.name.Rank;
  99 import eu.etaxonomy.cdm.model.name.TaxonName;
 100 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
 101 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
 102 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
 103 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
 104 import eu.etaxonomy.cdm.model.reference.Reference;
 105 import eu.etaxonomy.cdm.model.taxon.Classification;
 106 import eu.etaxonomy.cdm.model.taxon.HomotypicGroupTaxonComparator;
 107 import eu.etaxonomy.cdm.model.taxon.ITaxonTreeNode;
 108 import eu.etaxonomy.cdm.model.taxon.Synonym;
 109 import eu.etaxonomy.cdm.model.taxon.SynonymType;
 110 import eu.etaxonomy.cdm.model.taxon.Taxon;
 111 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
 112 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
 113 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
 114 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
 115 import eu.etaxonomy.cdm.persistence.dao.initializer.AbstractBeanInitializer;
 116 import eu.etaxonomy.cdm.persistence.dao.name.ITaxonNameDao;
 117 import eu.etaxonomy.cdm.persistence.dao.occurrence.IOccurrenceDao;
 118 import eu.etaxonomy.cdm.persistence.dao.taxon.IClassificationDao;
 119 import eu.etaxonomy.cdm.persistence.dao.taxon.ITaxonDao;
 120 import eu.etaxonomy.cdm.persistence.dao.taxon.ITaxonNodeDao;
 121 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
 122 import eu.etaxonomy.cdm.persistence.query.MatchMode;
 123 import eu.etaxonomy.cdm.persistence.query.OrderHint;
 124 import eu.etaxonomy.cdm.persistence.query.OrderHint.SortOrder;
 125 import eu.etaxonomy.cdm.persistence.query.TaxonTitleType;
 126 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
 127
 128
 129 /**
 130  * @author a.kohlbecker
 131  * @since 10.09.2010
 132  */
 133 @Service
 134 @Transactional(readOnly = true)
 135 public class TaxonServiceImpl
 136             extends IdentifiableServiceBase<TaxonBase,ITaxonDao>
 137             implements ITaxonService{
 138
 139     private static final Logger logger = Logger.getLogger(TaxonServiceImpl.class);
 140
 141     public static final String POTENTIAL_COMBINATION_NAMESPACE = "Potential combination";
 142
 143     public static final String INFERRED_EPITHET_NAMESPACE = "Inferred epithet";
 144
 145     public static final String INFERRED_GENUS_NAMESPACE = "Inferred genus";
 146
 147     @Autowired
 148     private ITaxonNodeDao taxonNodeDao;
 149
 150     @Autowired
 151     private ITaxonNameDao nameDao;
 152
 153     @Autowired
 154     private INameService nameService;
 155
 156     @Autowired
 157     private IOccurrenceService occurrenceService;
 158
 159     @Autowired
 160     private ITaxonNodeService nodeService;
 161
 162     @Autowired
 163     private IDescriptionService descriptionService;
 164 //
 165 //    @Autowired
 166 //    private IOrderedTermVocabularyDao orderedVocabularyDao;
 167
 168     @Autowired
 169     private IOccurrenceDao occurrenceDao;
 170
 171     @Autowired
 172     private IClassificationDao classificationDao;
 173
 174     @Autowired
 175     private AbstractBeanInitializer beanInitializer;
 176
 177     @Autowired
 178     private ILuceneIndexToolProvider luceneIndexToolProvider;
 179
 180 //************************ CONSTRUCTOR ****************************/
 181     public TaxonServiceImpl(){
 182         if (logger.isDebugEnabled()) { logger.debug("Load TaxonService Bean"); }
 183     }
 184
 185 // ****************************** METHODS ********************************/
 186
 187
 188     /**
 189      * {@inheritDoc}
 190      */
 191     @Override
 192     public TaxonBase load(UUID uuid, boolean includeUnpublished, List<String> propertyPaths) {
 193         return dao.load(uuid, includeUnpublished, propertyPaths);
 194     }
 195
 196     @Override
 197     public List<TaxonBase> searchByName(String name, boolean includeUnpublished, Reference sec) {
 198         return dao.getTaxaByName(name, includeUnpublished, sec);
 199     }
 200
 201     @Override
 202     @Transactional(readOnly = false)
 203     public UpdateResult swapSynonymAndAcceptedTaxon(Synonym synonym, Taxon acceptedTaxon){
 204         UpdateResult result = new UpdateResult();
 205         TaxonName synonymName = synonym.getName();
 206         synonymName.removeTaxonBase(synonym);
 207         TaxonName taxonName = acceptedTaxon.getName();
 208         taxonName.removeTaxonBase(acceptedTaxon);
 209
 210         synonym.setName(taxonName);
 211         synonym.setTitleCache(null, false);
 212         synonym.getTitleCache();
 213         acceptedTaxon.setName(synonymName);
 214         acceptedTaxon.setTitleCache(null, false);
 215         acceptedTaxon.getTitleCache();
 216         saveOrUpdate(synonym);
 217         saveOrUpdate(acceptedTaxon);
 218         result.addUpdatedObject(acceptedTaxon);
 219         result.addUpdatedObject(synonym);
 220                 return result;
 221
 222         // the accepted taxon needs a new uuid because the concept has changed
 223         // FIXME this leads to an error "HibernateException: immutable natural identifier of an instance of eu.etaxonomy.cdm.model.taxon.Taxon was altered"
 224         //acceptedTaxon.setUuid(UUID.randomUUID());
 225     }
 226
 227
 228     @Override
 229     @Transactional(readOnly = false)
 230     public UpdateResult changeSynonymToAcceptedTaxon(Synonym synonym, Taxon acceptedTaxon, boolean deleteSynonym) {
 231         UpdateResult result = new UpdateResult();
 232         TaxonName acceptedName = acceptedTaxon.getName();
 233         TaxonName synonymName = synonym.getName();
 234         HomotypicalGroup synonymHomotypicGroup = synonymName.getHomotypicalGroup();
 235
 236         //check synonym is not homotypic
 237         if (acceptedName.getHomotypicalGroup().equals(synonymHomotypicGroup)){
 238             String message = "The accepted taxon and the synonym are part of the same homotypical group and therefore can not be both accepted.";
 239             result.addException(new HomotypicalGroupChangeException(message));
 240             result.setAbort();
 241             return result;
 242         }
 243
 244         Taxon newAcceptedTaxon = Taxon.NewInstance(synonymName, acceptedTaxon.getSec());
 245         dao.save(newAcceptedTaxon);
 246         result.setCdmEntity(newAcceptedTaxon);
 247         SynonymType relTypeForGroup = SynonymType.HOMOTYPIC_SYNONYM_OF();
 248         List<Synonym> heteroSynonyms = acceptedTaxon.getSynonymsInGroup(synonymHomotypicGroup);
 249
 250         for (Synonym heteroSynonym : heteroSynonyms){
 251             if (synonym.equals(heteroSynonym)){
 252                 acceptedTaxon.removeSynonym(heteroSynonym, false);
 253             }else{
 254                 //move synonyms in same homotypic group to new accepted taxon
 255                 newAcceptedTaxon.addSynonym(heteroSynonym, relTypeForGroup);
 256             }
 257         }
 258         dao.saveOrUpdate(acceptedTaxon);
 259         result.addUpdatedObject(acceptedTaxon);
 260         if (deleteSynonym){
 261
 262             try {
 263                 this.dao.flush();
 264                 SynonymDeletionConfigurator config = new SynonymDeletionConfigurator();
 265                 config.setDeleteNameIfPossible(false);
 266                 this.deleteSynonym(synonym, config);
 267
 268             } catch (Exception e) {
 269                 result.addException(e);
 270             }
 271         }
 272
 273         return result;
 274     }
 275
 276     @Override
 277     @Transactional(readOnly = false)
 278     public UpdateResult changeSynonymToAcceptedTaxon(UUID synonymUuid,
 279             UUID acceptedTaxonUuid,
 280             UUID newParentNodeUuid,
 281             boolean deleteSynonym)  {
 282         UpdateResult result = new UpdateResult();
 283         Synonym synonym = CdmBase.deproxy(dao.load(synonymUuid), Synonym.class);
 284         Taxon acceptedTaxon = CdmBase.deproxy(dao.load(acceptedTaxonUuid), Taxon.class);
 285         result =  changeSynonymToAcceptedTaxon(synonym, acceptedTaxon, deleteSynonym);
 286         Taxon newTaxon = (Taxon)result.getCdmEntity();
 287         TaxonNode newParentNode = taxonNodeDao.load(newParentNodeUuid);
 288         TaxonNode newNode = newParentNode.addChildTaxon(newTaxon, null, null);
 289         taxonNodeDao.save(newNode);
 290         result.addUpdatedObject(newTaxon);
 291         result.addUpdatedObject(acceptedTaxon);
 292         result.setCdmEntity(newNode);
 293         return result;
 294     }
 295
 296
 297
 298
 299     @Override
 300     @Transactional(readOnly = false)
 301     public UpdateResult changeSynonymToRelatedTaxon(UUID synonymUuid,
 302             UUID toTaxonUuid,
 303             TaxonRelationshipType taxonRelationshipType,
 304             Reference citation,
 305             String microcitation){
 306
 307         UpdateResult result = new UpdateResult();
 308         Taxon toTaxon = (Taxon) dao.load(toTaxonUuid);
 309         Synonym synonym = (Synonym) dao.load(synonymUuid);
 310         result = changeSynonymToRelatedTaxon(synonym, toTaxon, taxonRelationshipType, citation, microcitation);
 311         Taxon relatedTaxon = (Taxon)result.getCdmEntity();
 312 //        result.setCdmEntity(relatedTaxon);
 313         result.addUpdatedObject(relatedTaxon);
 314         result.addUpdatedObject(toTaxon);
 315         return result;
 316     }
 317
 318     @Override
 319     @Transactional(readOnly = false)
 320     public UpdateResult changeSynonymToRelatedTaxon(Synonym synonym, Taxon toTaxon, TaxonRelationshipType taxonRelationshipType, Reference citation, String microcitation){
 321         // Get name from synonym
 322         if (synonym == null){
 323             return null;
 324         }
 325
 326         UpdateResult result = new UpdateResult();
 327
 328         TaxonName synonymName = synonym.getName();
 329
 330       /*  // remove synonym from taxon
 331         toTaxon.removeSynonym(synonym);
 332 */
 333         // Create a taxon with synonym name
 334         Taxon fromTaxon = Taxon.NewInstance(synonymName, null);
 335         save(fromTaxon);
 336         fromTaxon.setAppendedPhrase(synonym.getAppendedPhrase());
 337
 338         // Add taxon relation
 339         fromTaxon.addTaxonRelation(toTaxon, taxonRelationshipType, citation, microcitation);
 340         result.setCdmEntity(fromTaxon);
 341         // since we are swapping names, we have to detach the name from the synonym completely.
 342         // Otherwise the synonym will still be in the list of typified names.
 343        // synonym.getName().removeTaxonBase(synonym);
 344         result.includeResult(this.deleteSynonym(synonym, null));
 345
 346         return result;
 347     }
 348
 349     @Transactional(readOnly = false)
 350     @Override
 351     public void changeHomotypicalGroupOfSynonym(Synonym synonym, HomotypicalGroup newHomotypicalGroup,
 352             Taxon targetTaxon, boolean setBasionymRelationIfApplicable){
 353         // Get synonym name
 354         TaxonName synonymName = synonym.getName();
 355         HomotypicalGroup oldHomotypicalGroup = synonymName.getHomotypicalGroup();
 356
 357         // Switch groups
 358         oldHomotypicalGroup.removeTypifiedName(synonymName, false);
 359         newHomotypicalGroup.addTypifiedName(synonymName);
 360
 361         //remove existing basionym relationships
 362         synonymName.removeBasionyms();
 363
 364         //add basionym relationship
 365         if (setBasionymRelationIfApplicable){
 366             Set<TaxonName> basionyms = newHomotypicalGroup.getBasionyms();
 367             for (TaxonName basionym : basionyms){
 368                 synonymName.addBasionym(basionym);
 369             }
 370         }
 371
 372         //set synonym relationship correctly
 373         Taxon acceptedTaxon = synonym.getAcceptedTaxon();
 374
 375         boolean hasNewTargetTaxon = targetTaxon != null && !targetTaxon.equals(acceptedTaxon);
 376         if (acceptedTaxon != null){
 377
 378             HomotypicalGroup acceptedGroup = acceptedTaxon.getHomotypicGroup();
 379             boolean isHomotypicToTaxon = acceptedGroup.equals(newHomotypicalGroup);
 380             SynonymType newRelationType = isHomotypicToTaxon? SynonymType.HOMOTYPIC_SYNONYM_OF() : SynonymType.HETEROTYPIC_SYNONYM_OF();
 381             synonym.setType(newRelationType);
 382
 383             if (hasNewTargetTaxon){
 384                 acceptedTaxon.removeSynonym(synonym, false);
 385             }
 386         }
 387         if (hasNewTargetTaxon ){
 388             @SuppressWarnings("null")
 389             HomotypicalGroup acceptedGroup = targetTaxon.getHomotypicGroup();
 390             boolean isHomotypicToTaxon = acceptedGroup.equals(newHomotypicalGroup);
 391             SynonymType relType = isHomotypicToTaxon? SynonymType.HOMOTYPIC_SYNONYM_OF() : SynonymType.HETEROTYPIC_SYNONYM_OF();
 392             targetTaxon.addSynonym(synonym, relType);
 393         }
 394
 395     }
 396
 397     @Override
 398     @Transactional(readOnly = false)
 399     public void updateTitleCache(Class<? extends TaxonBase> clazz, Integer stepSize, IIdentifiableEntityCacheStrategy<TaxonBase> cacheStrategy, IProgressMonitor monitor) {
 400         if (clazz == null){
 401             clazz = TaxonBase.class;
 402         }
 403         super.updateTitleCacheImpl(clazz, stepSize, cacheStrategy, monitor);
 404     }
 405
 406     @Override
 407     @Autowired
 408     protected void setDao(ITaxonDao dao) {
 409         this.dao = dao;
 410     }
 411
 412     @Override
 413     public Pager<TaxonBase> findTaxaByName(Class<? extends TaxonBase> clazz, String uninomial,  String infragenericEpithet, String specificEpithet,     String infraspecificEpithet, String authorship, Rank rank, Integer pageSize,Integer pageNumber) {
 414         long numberOfResults = dao.countTaxaByName(clazz, uninomial, infragenericEpithet, specificEpithet, infraspecificEpithet, rank);
 415
 416         List<TaxonBase> results = new ArrayList<>();
 417         if(numberOfResults > 0) { // no point checking again
 418             results = dao.findTaxaByName(clazz, uninomial, infragenericEpithet, specificEpithet, infraspecificEpithet, authorship, rank, pageSize, pageNumber);
 419         }
 420
 421         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 422     }
 423
 424     @Override
 425     public List<TaxonBase> listTaxaByName(Class<? extends TaxonBase> clazz, String uninomial, String infragenericEpithet, String specificEpithet,       String infraspecificEpithet, String authorship, Rank rank, Integer pageSize,Integer pageNumber) {
 426         long numberOfResults = dao.countTaxaByName(clazz, uninomial, infragenericEpithet, specificEpithet, infraspecificEpithet, rank);
 427
 428         List<TaxonBase> results = new ArrayList<>();
 429         if(numberOfResults > 0) { // no point checking again
 430             results = dao.findTaxaByName(clazz, uninomial, infragenericEpithet, specificEpithet, infraspecificEpithet, authorship, rank, pageSize, pageNumber);
 431         }
 432
 433         return results;
 434     }
 435
 436     @Override
 437     public List<TaxonRelationship> listToTaxonRelationships(Taxon taxon, TaxonRelationshipType type,
 438             boolean includeUnpublished, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths){
 439         long numberOfResults = dao.countTaxonRelationships(taxon, type, includeUnpublished, TaxonRelationship.Direction.relatedTo);
 440
 441         List<TaxonRelationship> results = new ArrayList<>();
 442         if(numberOfResults > 0) { // no point checking again
 443             results = dao.getTaxonRelationships(taxon, type, includeUnpublished, pageSize, pageNumber, orderHints, propertyPaths, TaxonRelationship.Direction.relatedTo);
 444         }
 445         return results;
 446     }
 447
 448     @Override
 449     public Pager<TaxonRelationship> pageToTaxonRelationships(Taxon taxon, TaxonRelationshipType type,
 450             boolean includeUnpublished, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 451         long numberOfResults = dao.countTaxonRelationships(taxon, type, includeUnpublished, TaxonRelationship.Direction.relatedTo);
 452
 453         List<TaxonRelationship> results = new ArrayList<>();
 454         if(numberOfResults > 0) { // no point checking again
 455             results = dao.getTaxonRelationships(taxon, type, includeUnpublished, pageSize, pageNumber, orderHints, propertyPaths, TaxonRelationship.Direction.relatedTo);
 456         }
 457         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 458     }
 459
 460     @Override
 461     public List<TaxonRelationship> listFromTaxonRelationships(Taxon taxon, TaxonRelationshipType type,
 462             boolean includeUnpublished, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths){
 463         long numberOfResults = dao.countTaxonRelationships(taxon, type, includeUnpublished, TaxonRelationship.Direction.relatedFrom);
 464
 465         List<TaxonRelationship> results = new ArrayList<>();
 466         if(numberOfResults > 0) { // no point checking again
 467             results = dao.getTaxonRelationships(taxon, type, includeUnpublished, pageSize, pageNumber, orderHints, propertyPaths, TaxonRelationship.Direction.relatedFrom);
 468         }
 469         return results;
 470     }
 471
 472     @Override
 473     public Pager<TaxonRelationship> pageFromTaxonRelationships(Taxon taxon, TaxonRelationshipType type,
 474             boolean includeUnpublished, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 475         long numberOfResults = dao.countTaxonRelationships(taxon, type, includeUnpublished, TaxonRelationship.Direction.relatedFrom);
 476
 477         List<TaxonRelationship> results = new ArrayList<>();
 478         if(numberOfResults > 0) { // no point checking again
 479             results = dao.getTaxonRelationships(taxon, type, includeUnpublished, pageSize, pageNumber, orderHints, propertyPaths, TaxonRelationship.Direction.relatedFrom);
 480         }
 481         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 482     }
 483
 484     @Override
 485     public List<TaxonRelationship> listTaxonRelationships(Set<TaxonRelationshipType> types,
 486             Integer pageSize, Integer pageStart, List<OrderHint> orderHints, List<String> propertyPaths) {
 487         Long numberOfResults = dao.countTaxonRelationships(types);
 488
 489         List<TaxonRelationship> results = new ArrayList<>();
 490         if(numberOfResults > 0) {
 491             results = dao.getTaxonRelationships(types, pageSize, pageStart, orderHints, propertyPaths);
 492         }
 493         return results;
 494     }
 495
 496     @Override
 497     public Taxon findAcceptedTaxonFor(UUID synonymUuid, UUID classificationUuid,
 498             boolean includeUnpublished, List<String> propertyPaths) throws UnpublishedException{
 499
 500         Synonym synonym = null;
 501
 502         try {
 503             synonym = (Synonym) dao.load(synonymUuid);
 504             checkPublished(synonym, includeUnpublished, "Synoym is unpublished.");
 505         } catch (ClassCastException e){
 506             throw new EntityNotFoundException("The TaxonBase entity referenced by " + synonymUuid + " is not a Synonmy");
 507         } catch (NullPointerException e){
 508             throw new EntityNotFoundException("No TaxonBase entity found for " + synonymUuid);
 509         }
 510
 511         Classification classificationFilter = null;
 512         if(classificationUuid != null){
 513             try {
 514                 classificationFilter = classificationDao.load(classificationUuid);
 515             } catch (NullPointerException e){
 516                 //TODO not sure, why an NPE should be thrown in the above load method
 517                 throw new EntityNotFoundException("No Classification entity found for " + classificationUuid);
 518             }
 519             if(classificationFilter == null){
 520                 throw new EntityNotFoundException("No Classification entity found for " + classificationUuid);
 521             }
 522         }
 523
 524         long count = dao.countAcceptedTaxonFor(synonym, classificationFilter) ;
 525         if(count > 0){
 526             Taxon result = dao.acceptedTaxonFor(synonym, classificationFilter, propertyPaths);
 527             checkPublished(result, includeUnpublished, "Accepted taxon unpublished");
 528             return result;
 529         }else{
 530             return null;
 531         }
 532     }
 533
 534     @Override
 535     public Set<Taxon> listRelatedTaxa(Taxon taxon, Set<TaxonRelationshipEdge> includeRelationships, Integer maxDepth,
 536             boolean includeUnpublished, Integer limit, Integer start, List<String> propertyPaths) {
 537
 538         Set<Taxon> relatedTaxa = collectRelatedTaxa(taxon, includeRelationships, new HashSet<>(), includeUnpublished, maxDepth);
 539         relatedTaxa.remove(taxon);
 540         beanInitializer.initializeAll(relatedTaxa, propertyPaths);
 541         return relatedTaxa;
 542     }
 543
 544
 545     /**
 546      * Recursively collect related taxa for the given <code>taxon</code> . The returned list will also include the
 547      *  <code>taxon</code> supplied as parameter.
 548      *
 549      * @param taxon
 550      * @param includeRelationships
 551      * @param taxa
 552      * @param maxDepth can be <code>null</code> for infinite depth
 553      * @return
 554      */
 555     private Set<Taxon> collectRelatedTaxa(Taxon taxon, Set<TaxonRelationshipEdge> includeRelationships, Set<Taxon> taxa,
 556             boolean includeUnpublished, Integer maxDepth) {
 557
 558         if(taxa.isEmpty()) {
 559             taxa.add(taxon);
 560         }
 561
 562         if(includeRelationships.isEmpty()){
 563             return taxa;
 564         }
 565
 566         if(maxDepth != null) {
 567             maxDepth--;
 568         }
 569         if(logger.isDebugEnabled()){
 570             logger.debug("collecting related taxa for " + taxon + " with maxDepth=" + maxDepth);
 571         }
 572         List<TaxonRelationship> taxonRelationships = dao.getTaxonRelationships(taxon,
 573                 (Set<TaxonRelationshipType>)null, includeUnpublished, null, null, null, null, null);
 574         for (TaxonRelationship taxRel : taxonRelationships) {
 575
 576             // skip invalid data
 577             if (taxRel.getToTaxon() == null || taxRel.getFromTaxon() == null || taxRel.getType() == null) {
 578                 continue;
 579             }
 580             // filter by includeRelationships
 581             for (TaxonRelationshipEdge relationshipEdgeFilter : includeRelationships) {
 582                 if ( relationshipEdgeFilter.getRelationshipTypes().equals(taxRel.getType()) ) {
 583                     if (relationshipEdgeFilter.getDirections().contains(Direction.relatedTo) && !taxa.contains(taxRel.getToTaxon())) {
 584                         if(logger.isDebugEnabled()){
 585                             logger.debug(maxDepth + ": " + taxon.getTitleCache() + " --[" + taxRel.getType().getLabel() + "]--> " + taxRel.getToTaxon().getTitleCache());
 586                         }
 587                         taxa.add(taxRel.getToTaxon());
 588                         if(maxDepth == null || maxDepth > 0) {
 589                             taxa.addAll(collectRelatedTaxa(taxRel.getToTaxon(), includeRelationships, taxa, includeUnpublished, maxDepth));
 590                         }
 591                     }
 592                     if(relationshipEdgeFilter.getDirections().contains(Direction.relatedFrom) && !taxa.contains(taxRel.getFromTaxon())) {
 593                         taxa.add(taxRel.getFromTaxon());
 594                         if(logger.isDebugEnabled()){
 595                             logger.debug(maxDepth + ": " +taxRel.getFromTaxon().getTitleCache() + " --[" + taxRel.getType().getLabel() + "]--> " + taxon.getTitleCache() );
 596                         }
 597                         if(maxDepth == null || maxDepth > 0) {
 598                             taxa.addAll(collectRelatedTaxa(taxRel.getFromTaxon(), includeRelationships, taxa, includeUnpublished, maxDepth));
 599                         }
 600                     }
 601                 }
 602             }
 603         }
 604         return taxa;
 605     }
 606
 607     @Override
 608     public Pager<Synonym> getSynonyms(Taxon taxon,      SynonymType type, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 609         Long numberOfResults = dao.countSynonyms(taxon, type);
 610
 611         List<Synonym> results = new ArrayList<>();
 612         if(numberOfResults > 0) { // no point checking again
 613             results = dao.getSynonyms(taxon, type, pageSize, pageNumber, orderHints, propertyPaths);
 614         }
 615
 616         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);
 617     }
 618
 619     @Override
 620     public List<List<Synonym>> getSynonymsByHomotypicGroup(Taxon taxon, List<String> propertyPaths){
 621         List<List<Synonym>> result = new ArrayList<>();
 622         taxon = (Taxon)dao.load(taxon.getUuid(), propertyPaths);
 623         HomotypicGroupTaxonComparator comparator = new HomotypicGroupTaxonComparator(taxon);
 624
 625
 626         //homotypic
 627         result.add(taxon.getHomotypicSynonymsByHomotypicGroup(comparator));
 628
 629         //heterotypic
 630         List<HomotypicalGroup> homotypicalGroups = taxon.getHeterotypicSynonymyGroups();  //currently the list is sorted by the Taxon.defaultTaxonComparator
 631         for(HomotypicalGroup homotypicalGroup : homotypicalGroups){
 632             result.add(taxon.getSynonymsInGroup(homotypicalGroup, comparator));
 633         }
 634
 635         return result;
 636
 637     }
 638
 639     @Override
 640     public List<Synonym> getHomotypicSynonymsByHomotypicGroup(Taxon taxon, List<String> propertyPaths){
 641         Taxon t = (Taxon)dao.load(taxon.getUuid(), propertyPaths);
 642         HomotypicGroupTaxonComparator comparator = new HomotypicGroupTaxonComparator(taxon);
 643
 644         return t.getHomotypicSynonymsByHomotypicGroup(comparator);
 645     }
 646
 647     @Override
 648     public List<List<Synonym>> getHeterotypicSynonymyGroups(Taxon taxon, List<String> propertyPaths){
 649         Taxon t = (Taxon)dao.load(taxon.getUuid(), propertyPaths);
 650         List<HomotypicalGroup> homotypicalGroups = t.getHeterotypicSynonymyGroups();
 651         List<List<Synonym>> heterotypicSynonymyGroups = new ArrayList<>(homotypicalGroups.size());
 652         for(HomotypicalGroup homotypicalGroup : homotypicalGroups){
 653             heterotypicSynonymyGroups.add(t.getSynonymsInGroup(homotypicalGroup));
 654         }
 655         return heterotypicSynonymyGroups;
 656     }
 657
 658     @Override
 659     public List<UuidAndTitleCache<? extends IdentifiableEntity>> findTaxaAndNamesForEditor(IFindTaxaAndNamesConfigurator config){
 660
 661         if (config.isDoSynonyms() || config.isDoTaxa() || config.isDoNamesWithoutTaxa() || config.isDoTaxaByCommonNames()){
 662                 return dao.getTaxaByNameForEditor(config.isDoTaxa(), config.isDoSynonyms(), config.isDoNamesWithoutTaxa(),
 663                         config.isDoMisappliedNames(), config.isDoTaxaByCommonNames(), config.isIncludeUnpublished(),
 664                         config.getTitleSearchStringSqlized(), config.getClassification(), config.getSubtree(),
 665                         config.getMatchMode(), config.getNamedAreas(), config.getOrder());
 666         }else{
 667             return new ArrayList<>();
 668         }
 669     }
 670
 671     @Override
 672     public Pager<IdentifiableEntity> findTaxaAndNames(IFindTaxaAndNamesConfigurator configurator) {
 673
 674         List<IdentifiableEntity> results = new ArrayList<>();
 675         long numberOfResults = 0; // overall number of results (as opposed to number of results per page)
 676         List<TaxonBase> taxa = null;
 677
 678         // Taxa and synonyms
 679         long numberTaxaResults = 0L;
 680
 681         List<String> propertyPath = new ArrayList<>();
 682         if(configurator.getTaxonPropertyPath() != null){
 683             propertyPath.addAll(configurator.getTaxonPropertyPath());
 684         }
 685
 686         if (configurator.isDoMisappliedNames() || configurator.isDoSynonyms() || configurator.isDoTaxa() || configurator.isDoTaxaByCommonNames()){
 687             if(configurator.getPageSize() != null){ // no point counting if we need all anyway
 688                 numberTaxaResults =
 689                     dao.countTaxaByName(configurator.isDoTaxa(),configurator.isDoSynonyms(), configurator.isDoMisappliedNames(),
 690                         configurator.isDoTaxaByCommonNames(), configurator.isDoIncludeAuthors(), configurator.getTitleSearchStringSqlized(),
 691                         configurator.getClassification(), configurator.getSubtree(), configurator.getMatchMode(),
 692                         configurator.getNamedAreas(), configurator.isIncludeUnpublished());
 693             }
 694
 695             if(configurator.getPageSize() == null || numberTaxaResults > configurator.getPageSize() * configurator.getPageNumber()){ // no point checking again if less results
 696                 taxa = dao.getTaxaByName(configurator.isDoTaxa(), configurator.isDoSynonyms(),
 697                     configurator.isDoMisappliedNames(), configurator.isDoTaxaByCommonNames(), configurator.isDoIncludeAuthors(),
 698                     configurator.getTitleSearchStringSqlized(), configurator.getClassification(), configurator.getSubtree(),
 699                     configurator.getMatchMode(), configurator.getNamedAreas(), configurator.isIncludeUnpublished(),
 700                     configurator.getOrder(), configurator.getPageSize(), configurator.getPageNumber(), propertyPath);
 701             }
 702        }
 703
 704         if (logger.isDebugEnabled()) { logger.debug(numberTaxaResults + " matching taxa counted"); }
 705
 706         if(taxa != null){
 707             results.addAll(taxa);
 708         }
 709
 710         numberOfResults += numberTaxaResults;
 711
 712         // Names without taxa
 713         if (configurator.isDoNamesWithoutTaxa()) {
 714             int numberNameResults = 0;
 715
 716             List<TaxonName> names =
 717                 nameDao.findByName(configurator.isDoIncludeAuthors(), configurator.getTitleSearchStringSqlized(), configurator.getMatchMode(),
 718                         configurator.getPageSize(), configurator.getPageNumber(), null, configurator.getTaxonNamePropertyPath());
 719             if (logger.isDebugEnabled()) { logger.debug(names.size() + " matching name(s) found"); }
 720             if (names.size() > 0) {
 721                 for (TaxonName taxonName : names) {
 722                     if (taxonName.getTaxonBases().size() == 0) {
 723                         results.add(taxonName);
 724                         numberNameResults++;
 725                     }
 726                 }
 727                 if (logger.isDebugEnabled()) { logger.debug(numberNameResults + " matching name(s) without taxa found"); }
 728                 numberOfResults += numberNameResults;
 729             }
 730         }
 731
 732        return new DefaultPagerImpl<> (configurator.getPageNumber(), numberOfResults, configurator.getPageSize(), results);
 733     }
 734
 735     public List<UuidAndTitleCache<TaxonBase>> getTaxonUuidAndTitleCache(Integer limit, String pattern){
 736         return dao.getUuidAndTitleCache(limit, pattern);
 737     }
 738
 739     @Override
 740     public List<Media> listTaxonDescriptionMedia(Taxon taxon, Set<TaxonRelationshipEdge> includeRelationships, boolean limitToGalleries, List<String> propertyPath){
 741         return listMedia(taxon, includeRelationships, limitToGalleries, true, false, false, propertyPath);
 742     }
 743
 744     @Override
 745     public List<Media> listMedia(Taxon taxon, Set<TaxonRelationshipEdge> includeRelationships,
 746             Boolean limitToGalleries, Boolean includeTaxonDescriptions, Boolean includeOccurrences,
 747             Boolean includeTaxonNameDescriptions, List<String> propertyPath) {
 748
 749         //TODO let inherit
 750         boolean includeUnpublished = INCLUDE_UNPUBLISHED;
 751
 752     //    logger.setLevel(Level.TRACE);
 753 //        Logger.getLogger("org.hibernate.SQL").setLevel(Level.TRACE);
 754
 755         logger.trace("listMedia() - START");
 756
 757         Set<Taxon> taxa = new HashSet<>();
 758         List<Media> taxonMedia = new ArrayList<>();
 759         List<Media> nonImageGalleryImages = new ArrayList<>();
 760
 761         if (limitToGalleries == null) {
 762             limitToGalleries = false;
 763         }
 764
 765         // --- resolve related taxa
 766         if (includeRelationships != null && ! includeRelationships.isEmpty()) {
 767             logger.trace("listMedia() - resolve related taxa");
 768             taxa = listRelatedTaxa(taxon, includeRelationships, null, includeUnpublished, null, null, null);
 769         }
 770
 771         taxa.add((Taxon) dao.load(taxon.getUuid()));
 772
 773         if(includeTaxonDescriptions != null && includeTaxonDescriptions){
 774             logger.trace("listMedia() - includeTaxonDescriptions");
 775             List<TaxonDescription> taxonDescriptions = new ArrayList<>();
 776             // --- TaxonDescriptions
 777             for (Taxon t : taxa) {
 778                 taxonDescriptions.addAll(descriptionService.listTaxonDescriptions(t, null, null, null, null, propertyPath));
 779             }
 780             for (TaxonDescription taxonDescription : taxonDescriptions) {
 781                 if (!limitToGalleries || taxonDescription.isImageGallery()) {
 782                     for (DescriptionElementBase element : taxonDescription.getElements()) {
 783                         for (Media media : element.getMedia()) {
 784                             if(taxonDescription.isImageGallery()){
 785                                 taxonMedia.add(media);
 786                             }
 787                             else{
 788                                 nonImageGalleryImages.add(media);
 789                             }
 790                         }
 791                     }
 792                 }
 793             }
 794             //put images from image gallery first (#3242)
 795             taxonMedia.addAll(nonImageGalleryImages);
 796         }
 797
 798
 799         if(includeOccurrences != null && includeOccurrences) {
 800             logger.trace("listMedia() - includeOccurrences");
 801             Set<SpecimenOrObservationBase> specimensOrObservations = new HashSet<>();
 802             // --- Specimens
 803             for (Taxon t : taxa) {
 804                 specimensOrObservations.addAll(occurrenceDao.listByAssociatedTaxon(null, t, null, null, null, null));
 805             }
 806             for (SpecimenOrObservationBase<?> occurrence : specimensOrObservations) {
 807
 808 //              direct media removed from specimen #3597
 809 //              taxonMedia.addAll(occurrence.getMedia());
 810
 811                 // SpecimenDescriptions
 812                 Set<SpecimenDescription> specimenDescriptions = occurrence.getSpecimenDescriptions();
 813                 for (DescriptionBase<?> specimenDescription : specimenDescriptions) {
 814                     if (!limitToGalleries || specimenDescription.isImageGallery()) {
 815                         Set<DescriptionElementBase> elements = specimenDescription.getElements();
 816                         for (DescriptionElementBase element : elements) {
 817                             for (Media media : element.getMedia()) {
 818                                 taxonMedia.add(media);
 819                             }
 820                         }
 821                     }
 822                 }
 823
 824                 if (occurrence.isInstanceOf(DerivedUnit.class)) {
 825                     DerivedUnit derivedUnit = CdmBase.deproxy(occurrence, DerivedUnit.class);
 826                     // Collection
 827                     //TODO why may collections have media attached? #
 828                     if (derivedUnit.getCollection() != null){
 829                         taxonMedia.addAll(derivedUnit.getCollection().getMedia());
 830                     }
 831                 }
 832                 //media in hierarchy
 833                 taxonMedia.addAll(occurrenceService.getMediainHierarchy(occurrence, null, null, propertyPath).getRecords());
 834             }
 835         }
 836
 837         if(includeTaxonNameDescriptions != null && includeTaxonNameDescriptions) {
 838             logger.trace("listMedia() - includeTaxonNameDescriptions");
 839             // --- TaxonNameDescription
 840             Set<TaxonNameDescription> nameDescriptions = new HashSet<>();
 841             for (Taxon t : taxa) {
 842                 nameDescriptions .addAll(t.getName().getDescriptions());
 843             }
 844             for(TaxonNameDescription nameDescription: nameDescriptions){
 845                 if (!limitToGalleries || nameDescription.isImageGallery()) {
 846                     Set<DescriptionElementBase> elements = nameDescription.getElements();
 847                     for (DescriptionElementBase element : elements) {
 848                         for (Media media : element.getMedia()) {
 849                             taxonMedia.add(media);
 850                         }
 851                     }
 852                 }
 853             }
 854         }
 855
 856
 857         logger.trace("listMedia() - initialize");
 858         beanInitializer.initializeAll(taxonMedia, propertyPath);
 859
 860         logger.trace("listMedia() - END");
 861
 862         return taxonMedia;
 863     }
 864
 865     @Override
 866     public List<TaxonBase> findTaxaByID(Set<Integer> listOfIDs) {
 867         return this.dao.loadList(listOfIDs, null);
 868     }
 869
 870     @Override
 871     public TaxonBase findTaxonByUuid(UUID uuid, List<String> propertyPaths){
 872         return this.dao.findByUuid(uuid, null ,propertyPaths);
 873     }
 874
 875     @Override
 876     public long countSynonyms(boolean onlyAttachedToTaxon){
 877         return this.dao.countSynonyms(onlyAttachedToTaxon);
 878     }
 879
 880     @Override
 881     public List<TaxonName> findIdenticalTaxonNames(List<String> propertyPath) {
 882         return this.dao.findIdenticalTaxonNames(propertyPath);
 883     }
 884
 885     @Override
 886     @Transactional(readOnly=false)
 887     public DeleteResult deleteTaxon(UUID taxonUUID, TaxonDeletionConfigurator config, UUID classificationUuid)  {
 888
 889         if (config == null){
 890             config = new TaxonDeletionConfigurator();
 891         }
 892         Taxon taxon = (Taxon)dao.load(taxonUUID);
 893         DeleteResult result = new DeleteResult();
 894         if (taxon == null){
 895             result.setAbort();
 896             result.addException(new Exception ("The taxon was already deleted."));
 897             return result;
 898         }
 899         taxon = HibernateProxyHelper.deproxy(taxon);
 900         Classification classification = HibernateProxyHelper.deproxy(classificationDao.load(classificationUuid), Classification.class);
 901         result = isDeletable(taxonUUID, config);
 902
 903         if (result.isOk()){
 904             // --- DeleteSynonymRelations
 905             if (config.isDeleteSynonymRelations()){
 906                 boolean removeSynonymNameFromHomotypicalGroup = false;
 907                 // use tmp Set to avoid concurrent modification
 908                 Set<Synonym> synsToDelete = new HashSet<>();
 909                 synsToDelete.addAll(taxon.getSynonyms());
 910                 for (Synonym synonym : synsToDelete){
 911                     taxon.removeSynonym(synonym, removeSynonymNameFromHomotypicalGroup);
 912
 913                     // --- DeleteSynonymsIfPossible
 914                     if (config.isDeleteSynonymsIfPossible()){
 915                         //TODO which value
 916                         boolean newHomotypicGroupIfNeeded = true;
 917                         SynonymDeletionConfigurator synConfig = new SynonymDeletionConfigurator();
 918                         result.includeResult(deleteSynonym(synonym, synConfig));
 919                     }
 920                 }
 921             }
 922
 923             // --- DeleteTaxonRelationships
 924             if (! config.isDeleteTaxonRelationships()){
 925                 if (taxon.getTaxonRelations().size() > 0){
 926                     result.setAbort();
 927                     result.addException(new Exception("Taxon can't be deleted as it is related to another taxon. " +
 928                             "Remove taxon from all relations to other taxa prior to deletion."));
 929                 }
 930             } else{
 931                 TaxonDeletionConfigurator configRelTaxon = new TaxonDeletionConfigurator();
 932                 configRelTaxon.setDeleteTaxonNodes(false);
 933                 configRelTaxon.setDeleteConceptRelationships(true);
 934
 935                 for (TaxonRelationship taxRel: taxon.getTaxonRelations()){
 936                     if (config.isDeleteMisappliedNamesAndInvalidDesignations()
 937                             && taxRel.getType().isMisappliedNameOrInvalidDesignation()
 938                             && taxon.equals(taxRel.getToTaxon())){
 939                         this.deleteTaxon(taxRel.getFromTaxon().getUuid(), config, classificationUuid);
 940                     } else if (config.isDeleteConceptRelationships() && taxRel.getType().isConceptRelationship()){
 941
 942                         if (taxon.equals(taxRel.getToTaxon()) && isDeletable(taxRel.getFromTaxon().getUuid(), configRelTaxon).isOk()){
 943                             this.deleteTaxon(taxRel.getFromTaxon().getUuid(), configRelTaxon, classificationUuid);
 944                         }else if (isDeletable(taxRel.getToTaxon().getUuid(), configRelTaxon).isOk()){
 945                             this.deleteTaxon(taxRel.getToTaxon().getUuid(), configRelTaxon, classificationUuid);
 946                         }
 947                     }
 948                     taxon.removeTaxonRelation(taxRel);
 949                 }
 950             }
 951
 952             //          TaxonDescription
 953             if (config.isDeleteDescriptions()){
 954                 Set<TaxonDescription> descriptions = taxon.getDescriptions();
 955                 List<TaxonDescription> removeDescriptions = new ArrayList<>();
 956                 for (TaxonDescription desc: descriptions){
 957                     //TODO use description delete configurator ?
 958                     //FIXME check if description is ALWAYS deletable
 959                     if (desc.getDescribedSpecimenOrObservation() != null){
 960                         result.setAbort();
 961                         result.addException(new Exception("Taxon can't be deleted as it is used in a TaxonDescription" +
 962                                 " which also describes specimens or observations"));
 963                         break;
 964                     }
 965                     removeDescriptions.add(desc);
 966
 967
 968                 }
 969                 if (result.isOk()){
 970                     for (TaxonDescription desc: removeDescriptions){
 971                         taxon.removeDescription(desc);
 972                         descriptionService.delete(desc);
 973                     }
 974                 } else {
 975                     return result;
 976                 }
 977             }
 978
 979
 980          if (! config.isDeleteTaxonNodes() || (!config.isDeleteInAllClassifications() && classification == null && taxon.getTaxonNodes().size() > 1)){
 981              result.addException(new Exception( "Taxon can't be deleted as it is used in more than one classification."));
 982          }else{
 983              if (taxon.getTaxonNodes().size() != 0){
 984                 Set<TaxonNode> nodes = taxon.getTaxonNodes();
 985                 Iterator<TaxonNode> iterator = nodes.iterator();
 986                 TaxonNode node = null;
 987                 boolean deleteChildren;
 988                 if (config.getTaxonNodeConfig().getChildHandling().equals(ChildHandling.DELETE)){
 989                     deleteChildren = true;
 990                 }else {
 991                     deleteChildren = false;
 992                 }
 993                 boolean success = true;
 994                 if (!config.isDeleteInAllClassifications() && !(classification == null)){
 995                     while (iterator.hasNext()){
 996                         node = iterator.next();
 997                         if (node.getClassification().equals(classification)){
 998                             break;
 999                         }
1000                         node = null;
1001                     }
1002                     if (node != null){
1003                         HibernateProxyHelper.deproxy(node, TaxonNode.class);
1004                         success =taxon.removeTaxonNode(node, deleteChildren);
1005                         nodeService.delete(node);
1006                         result.addDeletedObject(node);
1007                     } else {
1008                         result.setError();
1009                         result.addException(new Exception("The taxon can not be deleted because it is not used in defined classification."));
1010                     }
1011                 } else if (config.isDeleteInAllClassifications()){
1012                     List<TaxonNode> nodesList = new ArrayList<>();
1013                     nodesList.addAll(taxon.getTaxonNodes());
1014                     for (ITaxonTreeNode treeNode: nodesList){
1015                         TaxonNode taxonNode = (TaxonNode) treeNode;
1016                         if(!deleteChildren){
1017                             Object[] childNodes = taxonNode.getChildNodes().toArray();
1018                             for (Object childNode: childNodes){
1019                                 TaxonNode childNodeCast = (TaxonNode) childNode;
1020                                 taxonNode.getParent().addChildNode(childNodeCast, childNodeCast.getReference(), childNodeCast.getMicroReference());
1021                             }
1022                         }
1023                     }
1024                     config.getTaxonNodeConfig().setDeleteElement(false);
1025                     DeleteResult resultNodes = nodeService.deleteTaxonNodes(nodesList, config);
1026                     if (!resultNodes.isOk()){
1027                         result.addExceptions(resultNodes.getExceptions());
1028                         result.setStatus(resultNodes.getStatus());
1029                     } else {
1030                         result.addUpdatedObjects(resultNodes.getUpdatedObjects());
1031                     }
1032                 }
1033                 if (!success){
1034                     result.setError();
1035                     result.addException(new Exception("The taxon can not be deleted because the taxon node can not be removed."));
1036                 }
1037             }
1038          }
1039          TaxonName name = taxon.getName();
1040          taxon.setName(null);
1041          this.saveOrUpdate(taxon);
1042
1043          if ((taxon.getTaxonNodes() == null || taxon.getTaxonNodes().size()== 0)  && result.isOk()){
1044              try{
1045                  dao.delete(taxon);
1046                  result.addDeletedObject(taxon);
1047              }catch(Exception e){
1048                  result.addException(e);
1049                  result.setError();
1050              }
1051          } else {
1052              result.setError();
1053              result.addException(new Exception("The Taxon can't be deleted because it is used in a classification."));
1054
1055          }
1056          //TaxonName
1057          if (config.isDeleteNameIfPossible() && result.isOk()){
1058             DeleteResult nameResult = new DeleteResult();
1059             //remove name if possible (and required)
1060             if (name != null ){
1061                 nameResult = nameService.delete(name.getUuid(), config.getNameDeletionConfig());
1062             }
1063             if (nameResult.isError() || nameResult.isAbort()){
1064                 result.addRelatedObject(name);
1065                 result.addExceptions(nameResult.getExceptions());
1066             }else{
1067                 result.includeResult(nameResult);
1068             }
1069          }
1070        }
1071
1072        return result;
1073
1074     }
1075
1076     @Override
1077     @Transactional(readOnly = false)
1078     public DeleteResult delete(UUID synUUID){
1079         Synonym syn = (Synonym)dao.load(synUUID);
1080         return this.deleteSynonym(syn, null);
1081     }
1082
1083     @Override
1084     @Transactional(readOnly = false)
1085     public DeleteResult deleteSynonym(UUID synonymUuid, SynonymDeletionConfigurator config) {
1086         return deleteSynonym((Synonym)dao.load(synonymUuid), config);
1087
1088     }
1089
1090
1091     @Override
1092     @Transactional(readOnly = false)
1093     public DeleteResult deleteSynonym(Synonym synonym, SynonymDeletionConfigurator config) {
1094         DeleteResult result = new DeleteResult();
1095         if (synonym == null){
1096                 result.setAbort();
1097                 result.addException(new Exception("The synonym was already deleted."));
1098                 return result;
1099         }
1100
1101         if (config == null){
1102             config = new SynonymDeletionConfigurator();
1103         }
1104
1105         result = isDeletable(synonym.getUuid(), config);
1106
1107         if (result.isOk()){
1108
1109             synonym = HibernateProxyHelper.deproxy(this.load(synonym.getUuid()), Synonym.class);
1110
1111             //remove synonym
1112             Taxon accTaxon = synonym.getAcceptedTaxon();
1113
1114             if (accTaxon != null){
1115                 accTaxon = HibernateProxyHelper.deproxy(accTaxon, Taxon.class);
1116                 accTaxon.removeSynonym(synonym, false);
1117                 this.saveOrUpdate(accTaxon);
1118                 result.addUpdatedObject(accTaxon);
1119             }
1120             this.saveOrUpdate(synonym);
1121             //#6281
1122             dao.flush();
1123
1124             TaxonName name = synonym.getName();
1125             synonym.setName(null);
1126
1127             dao.delete(synonym);
1128             result.addDeletedObject(synonym);
1129
1130             //remove name if possible (and required)
1131             if (name != null && config.isDeleteNameIfPossible()){
1132
1133                     DeleteResult nameDeleteResult = nameService.delete(name, config.getNameDeletionConfig());
1134                     if (nameDeleteResult.isAbort() || nameDeleteResult.isError()){
1135                         result.addExceptions(nameDeleteResult.getExceptions());
1136                         result.addRelatedObject(name);
1137                     }else{
1138                         result.addDeletedObject(name);
1139                     }
1140             }
1141
1142         }
1143         return result;
1144     }
1145
1146     @Override
1147     public List<TaxonName> findIdenticalTaxonNameIds(List<String> propertyPath) {
1148
1149         return this.dao.findIdenticalNamesNew(propertyPath);
1150     }
1151
1152
1153     @Override
1154     public Taxon findBestMatchingTaxon(String taxonName) {
1155         MatchingTaxonConfigurator config = MatchingTaxonConfigurator.NewInstance();
1156         config.setTaxonNameTitle(taxonName);
1157         return findBestMatchingTaxon(config);
1158     }
1159
1160     @Override
1161     public Taxon findBestMatchingTaxon(MatchingTaxonConfigurator config) {
1162
1163         Taxon bestCandidate = null;
1164         try{
1165             // 1. search for accepted taxa
1166             List<TaxonBase> taxonList = dao.findByNameTitleCache(true, false, config.isIncludeUnpublished(),
1167                     config.getTaxonNameTitle(), null, null, MatchMode.EXACT, null, null, 0, null, null);
1168             boolean bestCandidateMatchesSecUuid = false;
1169             boolean bestCandidateIsInClassification = false;
1170             int countEqualCandidates = 0;
1171             for(TaxonBase<?> taxonBaseCandidate : taxonList){
1172                 if(taxonBaseCandidate instanceof Taxon){
1173                     Taxon newCanditate = CdmBase.deproxy(taxonBaseCandidate, Taxon.class);
1174                     boolean newCandidateMatchesSecUuid = isMatchesSecUuid(newCanditate, config);
1175                     if (! newCandidateMatchesSecUuid && config.isOnlyMatchingSecUuid() ){
1176                         continue;
1177                     }else if(newCandidateMatchesSecUuid && ! bestCandidateMatchesSecUuid){
1178                         bestCandidate = newCanditate;
1179                         countEqualCandidates = 1;
1180                         bestCandidateMatchesSecUuid = true;
1181                         continue;
1182                     }
1183
1184                     boolean newCandidateInClassification = isInClassification(newCanditate, config);
1185                     if (! newCandidateInClassification && config.isOnlyMatchingClassificationUuid()){
1186                         continue;
1187                     }else if (newCandidateInClassification && ! bestCandidateIsInClassification){
1188                         bestCandidate = newCanditate;
1189                         countEqualCandidates = 1;
1190                         bestCandidateIsInClassification = true;
1191                         continue;
1192                     }
1193                     if (bestCandidate == null){
1194                         bestCandidate = newCanditate;
1195                         countEqualCandidates = 1;
1196                         continue;
1197                     }
1198
1199                 }else{  //not Taxon.class
1200                     continue;
1201                 }
1202                 countEqualCandidates++;
1203
1204             }
1205             if (bestCandidate != null){
1206                 if(countEqualCandidates > 1){
1207                     logger.info(countEqualCandidates + " equally matching TaxonBases found, using first accepted Taxon: " + bestCandidate.getTitleCache());
1208                     return bestCandidate;
1209                 } else {
1210                     logger.info("using accepted Taxon: " + bestCandidate.getTitleCache());
1211                     return bestCandidate;
1212                 }
1213             }
1214
1215
1216             // 2. search for synonyms
1217             if (config.isIncludeSynonyms()){
1218                 List<TaxonBase> synonymList = dao.findByNameTitleCache(false, true, config.isIncludeUnpublished(),
1219                         config.getTaxonNameTitle(), null, null, MatchMode.EXACT, null, null, 0, null, null);
1220                 for(TaxonBase taxonBase : synonymList){
1221                     if(taxonBase instanceof Synonym){
1222                         Synonym synonym = CdmBase.deproxy(taxonBase, Synonym.class);
1223                         bestCandidate = synonym.getAcceptedTaxon();
1224                         if(bestCandidate != null){
1225                             logger.info("using accepted Taxon " +  bestCandidate.getTitleCache() + " for synonym " + taxonBase.getTitleCache());
1226                             return bestCandidate;
1227                         }
1228                         //TODO extend method: search using treeUUID, using SecUUID, first find accepted then include synonyms until a matching taxon is found
1229                     }
1230                 }
1231             }
1232
1233         } catch (Exception e){
1234             logger.error(e);
1235             e.printStackTrace();
1236         }
1237
1238         return bestCandidate;
1239     }
1240
1241     private boolean isInClassification(Taxon taxon, MatchingTaxonConfigurator config) {
1242         UUID configClassificationUuid = config.getClassificationUuid();
1243         if (configClassificationUuid == null){
1244             return false;
1245         }
1246         for (TaxonNode node : taxon.getTaxonNodes()){
1247             UUID classUuid = node.getClassification().getUuid();
1248             if (configClassificationUuid.equals(classUuid)){
1249                 return true;
1250             }
1251         }
1252         return false;
1253     }
1254
1255     private boolean isMatchesSecUuid(Taxon taxon, MatchingTaxonConfigurator config) {
1256         UUID configSecUuid = config.getSecUuid();
1257         if (configSecUuid == null){
1258             return false;
1259         }
1260         UUID taxonSecUuid = (taxon.getSec() == null)? null : taxon.getSec().getUuid();
1261         return configSecUuid.equals(taxonSecUuid);
1262     }
1263
1264     @Override
1265     public Synonym findBestMatchingSynonym(String taxonName, boolean includeUnpublished) {
1266         List<TaxonBase> synonymList = dao.findByNameTitleCache(false, true, includeUnpublished, taxonName, null, null, MatchMode.EXACT, null, null, 0, null, null);
1267         if(! synonymList.isEmpty()){
1268             Synonym result = CdmBase.deproxy(synonymList.iterator().next(), Synonym.class);
1269             if(synonymList.size() == 1){
1270                 logger.info(synonymList.size() + " Synonym found " + result.getTitleCache() );
1271                 return result;
1272             } else {
1273                 logger.info("Several matching synonyms found. Using first: " +  result.getTitleCache());
1274                 return result;
1275             }
1276         }
1277         return null;
1278     }
1279
1280     @Override
1281     @Transactional(readOnly = false)
1282     public UpdateResult moveSynonymToAnotherTaxon(Synonym oldSynonym,
1283             Taxon newTaxon,
1284             boolean moveHomotypicGroup,
1285             SynonymType newSynonymType) throws HomotypicalGroupChangeException {
1286         return moveSynonymToAnotherTaxon(oldSynonym, newTaxon, moveHomotypicGroup,
1287                 newSynonymType,
1288                 oldSynonym.getSec(),
1289                 oldSynonym.getSecMicroReference(),
1290                 true);
1291     }
1292
1293     @Override
1294     @Transactional(readOnly = false)
1295     public UpdateResult moveSynonymToAnotherTaxon(Synonym oldSynonym,
1296             Taxon newTaxon,
1297             boolean moveHomotypicGroup,
1298             SynonymType newSynonymType,
1299             Reference newSecundum,
1300             String newSecundumDetail,
1301             boolean keepSecundumIfUndefined) throws HomotypicalGroupChangeException {
1302
1303         Synonym synonym = CdmBase.deproxy(dao.load(oldSynonym.getUuid()), Synonym.class);
1304         Taxon oldTaxon = CdmBase.deproxy(dao.load(synonym.getAcceptedTaxon().getUuid()), Taxon.class);
1305         //TODO what if there is no name ?? Concepts may be cached (e.g. via TCS import)
1306         TaxonName synonymName = synonym.getName();
1307         TaxonName fromTaxonName = oldTaxon.getName();
1308         //set default relationship type
1309         if (newSynonymType == null){
1310             newSynonymType = SynonymType.HETEROTYPIC_SYNONYM_OF();
1311         }
1312         boolean newRelTypeIsHomotypic = newSynonymType.equals(SynonymType.HOMOTYPIC_SYNONYM_OF());
1313
1314         HomotypicalGroup homotypicGroup = synonymName.getHomotypicalGroup();
1315         int hgSize = homotypicGroup.getTypifiedNames().size();
1316         boolean isSingleInGroup = !(hgSize > 1);
1317
1318         if (! isSingleInGroup){
1319             boolean isHomotypicToAccepted = synonymName.isHomotypic(fromTaxonName);
1320             boolean hasHomotypicSynonymRelatives = isHomotypicToAccepted ? hgSize > 2 : hgSize > 1;
1321             if (isHomotypicToAccepted){
1322                 String message = "Synonym is in homotypic group with accepted taxon%s. First remove synonym from homotypic group of accepted taxon before moving to other taxon.";
1323                 String homotypicRelatives = hasHomotypicSynonymRelatives ? " and other synonym(s)":"";
1324                 message = String.format(message, homotypicRelatives);
1325                 throw new HomotypicalGroupChangeException(message);
1326             }
1327             if (! moveHomotypicGroup){
1328                 String message = "Synonym is in homotypic group with other synonym(s). Either move complete homotypic group or remove synonym from homotypic group prior to moving to other taxon.";
1329                 throw new HomotypicalGroupChangeException(message);
1330             }
1331         }else{
1332             moveHomotypicGroup = true;  //single synonym always allows to moveCompleteGroup
1333         }
1334 //        Assert.assertTrue("Synonym can only be moved with complete homotypic group", moveHomotypicGroup);
1335
1336         UpdateResult result = new UpdateResult();
1337         //move all synonyms to new taxon
1338         List<Synonym> homotypicSynonyms = oldTaxon.getSynonymsInGroup(homotypicGroup);
1339         for (Synonym synRelation: homotypicSynonyms){
1340
1341             newTaxon = HibernateProxyHelper.deproxy(newTaxon, Taxon.class);
1342             oldTaxon = HibernateProxyHelper.deproxy(oldTaxon, Taxon.class);
1343             oldTaxon.removeSynonym(synRelation, false);
1344             newTaxon.addSynonym(synRelation, newSynonymType);
1345
1346             if (newSecundum != null || !keepSecundumIfUndefined){
1347                 synRelation.setSec(newSecundum);
1348             }
1349             if (newSecundumDetail != null || !keepSecundumIfUndefined){
1350                 synRelation.setSecMicroReference(newSecundumDetail);
1351             }
1352
1353             //set result  //why is this needed? Seems wrong to me (AM 10.10.2016)
1354             if (!synRelation.equals(oldSynonym)){
1355                 result.setError();
1356             }
1357         }
1358
1359         result.addUpdatedObject(oldTaxon);
1360         result.addUpdatedObject(newTaxon);
1361         saveOrUpdate(oldTaxon);
1362         saveOrUpdate(newTaxon);
1363
1364         return result;
1365     }
1366
1367     @Override
1368     public <T extends TaxonBase> List<UuidAndTitleCache<T>> getUuidAndTitleCache(Class<T> clazz, Integer limit, String pattern) {
1369         return dao.getUuidAndTitleCache(clazz, limit, pattern);
1370     }
1371
1372     @Override
1373     public Pager<SearchResult<TaxonBase>> findByFullText(
1374             Class<? extends TaxonBase> clazz, String queryString,
1375             Classification classification, TaxonNode subtree, boolean includeUnpublished, List<Language> languages,
1376             boolean highlightFragments, Integer pageSize, Integer pageNumber,
1377             List<OrderHint> orderHints, List<String> propertyPaths) throws IOException, LuceneParseException {
1378
1379         LuceneSearch luceneSearch = prepareFindByFullTextSearch(clazz, queryString, classification, subtree,
1380                 null, includeUnpublished, languages, highlightFragments, null);
1381
1382         // --- execute search
1383         TopGroups<BytesRef> topDocsResultSet;
1384         try {
1385             topDocsResultSet = luceneSearch.executeSearch(pageSize, pageNumber);
1386         } catch (ParseException e) {
1387             LuceneParseException luceneParseException = new LuceneParseException(e.getMessage());
1388             luceneParseException.setStackTrace(e.getStackTrace());
1389             throw luceneParseException;
1390         }
1391
1392         Map<CdmBaseType, String> idFieldMap = new HashMap<>();
1393         idFieldMap.put(CdmBaseType.TAXON, "id");
1394
1395         // ---  initialize taxa, thighlight matches ....
1396         ISearchResultBuilder searchResultBuilder = new SearchResultBuilder(luceneSearch, luceneSearch.getQuery());
1397         List<SearchResult<TaxonBase>> searchResults = searchResultBuilder.createResultSet(
1398                 topDocsResultSet, luceneSearch.getHighlightFields(), dao, idFieldMap, propertyPaths);
1399
1400         long totalHits = topDocsResultSet != null ? Long.valueOf(topDocsResultSet.totalGroupCount) : 0;
1401         return new DefaultPagerImpl<>(pageNumber, totalHits, pageSize, searchResults);
1402     }
1403
1404     @Override
1405     public Pager<SearchResult<TaxonBase>> findByDistribution(List<NamedArea> areaFilter, List<PresenceAbsenceTerm> statusFilter,
1406             Classification classification,
1407             Integer pageSize, Integer pageNumber,
1408             List<OrderHint> orderHints, List<String> propertyPaths) throws IOException, LuceneParseException {
1409
1410         LuceneSearch luceneSearch = prepareByDistributionSearch(areaFilter, statusFilter, classification);
1411
1412         // --- execute search
1413         TopGroups<BytesRef> topDocsResultSet;
1414         try {
1415             topDocsResultSet = luceneSearch.executeSearch(pageSize, pageNumber);
1416         } catch (ParseException e) {
1417             LuceneParseException luceneParseException = new LuceneParseException(e.getMessage());
1418             luceneParseException.setStackTrace(e.getStackTrace());
1419             throw luceneParseException;
1420         }
1421
1422         Map<CdmBaseType, String> idFieldMap = new HashMap<>();
1423         idFieldMap.put(CdmBaseType.TAXON, "id");
1424
1425         // ---  initialize taxa, thighlight matches ....
1426         ISearchResultBuilder searchResultBuilder = new SearchResultBuilder(luceneSearch, luceneSearch.getQuery());
1427         List<SearchResult<TaxonBase>> searchResults = searchResultBuilder.createResultSet(
1428                 topDocsResultSet, luceneSearch.getHighlightFields(), dao, idFieldMap, propertyPaths);
1429
1430         long totalHits = topDocsResultSet != null ? Long.valueOf(topDocsResultSet.totalGroupCount) : 0;
1431         return new DefaultPagerImpl<>(pageNumber, totalHits, pageSize, searchResults);
1432     }
1433
1434     /**
1435      * @param clazz
1436      * @param queryString
1437      * @param classification
1438      * @param includeUnpublished
1439      * @param languages
1440      * @param highlightFragments
1441      * @param sortFields TODO
1442      * @param directorySelectClass
1443      * @return
1444      */
1445     protected LuceneSearch prepareFindByFullTextSearch(Class<? extends CdmBase> clazz, String queryString,
1446             Classification classification, TaxonNode subtree, String className, boolean includeUnpublished, List<Language> languages,
1447             boolean highlightFragments, SortField[] sortFields) {
1448
1449         Builder finalQueryBuilder = new Builder();
1450         Builder textQueryBuilder = new Builder();
1451
1452         LuceneSearch luceneSearch = new LuceneSearch(luceneIndexToolProvider, GroupByTaxonClassBridge.GROUPBY_TAXON_FIELD, TaxonBase.class);
1453         QueryFactory taxonBaseQueryFactory = luceneIndexToolProvider.newQueryFactoryFor(TaxonBase.class);
1454
1455         if(sortFields == null){
1456             sortFields = new SortField[]{SortField.FIELD_SCORE, new SortField("titleCache__sort", SortField.Type.STRING, false)};
1457         }
1458         luceneSearch.setSortFields(sortFields);
1459
1460         // ---- search criteria
1461         luceneSearch.setCdmTypRestriction(clazz);
1462
1463         if(!StringUtils.isEmpty(queryString) && !queryString.equals("*") && !queryString.equals("?") ) {
1464             textQueryBuilder.add(taxonBaseQueryFactory.newTermQuery("titleCache", queryString), Occur.SHOULD);
1465             textQueryBuilder.add(taxonBaseQueryFactory.newDefinedTermQuery("name.rank", queryString, languages), Occur.SHOULD);
1466         }
1467         if(className != null){
1468             textQueryBuilder.add(taxonBaseQueryFactory.newTermQuery("classInfo.name", className, false), Occur.MUST);
1469         }
1470
1471         BooleanQuery textQuery = textQueryBuilder.build();
1472         if(textQuery.clauses().size() > 0) {
1473             finalQueryBuilder.add(textQuery, Occur.MUST);
1474         }
1475
1476         if(classification != null){
1477             finalQueryBuilder.add(taxonBaseQueryFactory.newEntityIdQuery("taxonNodes.classification.id", classification), Occur.MUST);
1478         }
1479         if(subtree != null){
1480             finalQueryBuilder.add(taxonBaseQueryFactory.newTermQuery("taxonNodes.treeIndex", subtree.treeIndexWc(), true), Occur.MUST);
1481         }
1482         if(!includeUnpublished)  {
1483             String accPublishParam = TaxonBase.ACC_TAXON_BRIDGE_PREFIX + AcceptedTaxonBridge.DOC_KEY_PUBLISH_SUFFIX;
1484             finalQueryBuilder.add(taxonBaseQueryFactory.newBooleanQuery(accPublishParam, true), Occur.MUST);
1485             finalQueryBuilder.add(taxonBaseQueryFactory.newBooleanQuery("publish", true), Occur.MUST);
1486         }
1487
1488         luceneSearch.setQuery(finalQueryBuilder.build());
1489
1490         if(highlightFragments){
1491             luceneSearch.setHighlightFields(taxonBaseQueryFactory.getTextFieldNamesAsArray());
1492         }
1493         return luceneSearch;
1494     }
1495
1496     /**
1497      * Uses org.apache.lucene.search.join.JoinUtil for query time joining, alternatively
1498      * the BlockJoinQuery could be used. The latter might be more memory save but has the
1499      * drawback of requiring to do the join an indexing time.
1500      * see  http://dev.e-taxonomy.eu/trac/wiki/LuceneNotes#JoinsinLucene for more information on this.
1501      *
1502      * Joins TaxonRelationShip with Taxon depending on the direction of the given edge:
1503      * <ul>
1504      * <li>direct, everted: {@link Direction.relatedTo}: TaxonRelationShip.relatedTo.id --&gt; Taxon.id </li>
1505      * <li>inverse: {@link Direction.relatedFrom}:  TaxonRelationShip.relatedFrom.id --&gt; Taxon.id </li>
1506      * <ul>
1507      * @param queryString
1508      * @param classification
1509      * @param languages
1510      * @param highlightFragments
1511      * @param sortFields TODO
1512      *
1513      * @return
1514      * @throws IOException
1515      */
1516     protected LuceneSearch prepareFindByTaxonRelationFullTextSearch(TaxonRelationshipEdge edge, String queryString,
1517             Classification classification, TaxonNode subtree, boolean includeUnpublished, List<Language> languages,
1518             boolean highlightFragments, SortField[] sortFields) throws IOException {
1519
1520         String fromField;
1521         String queryTermField;
1522         String toField = "id"; // TaxonBase.uuid
1523         String publishField;
1524         String publishFieldInvers;
1525
1526         if(edge.isBidirectional()){
1527             throw new RuntimeException("Bidirectional joining not supported!");
1528         }
1529         if(edge.isEvers()){
1530             fromField = "relatedFrom.id";
1531             queryTermField = "relatedFrom.titleCache";
1532             publishField = "relatedFrom.publish";
1533             publishFieldInvers = "relatedTo.publish";
1534         } else if(edge.isInvers()) {
1535             fromField = "relatedTo.id";
1536             queryTermField = "relatedTo.titleCache";
1537             publishField = "relatedTo.publish";
1538             publishFieldInvers = "relatedFrom.publish";
1539         } else {
1540             throw new RuntimeException("Invalid direction: " + edge.getDirections());
1541         }
1542
1543         Builder finalQueryBuilder = new Builder();
1544
1545         LuceneSearch luceneSearch = new LuceneSearch(luceneIndexToolProvider, GroupByTaxonClassBridge.GROUPBY_TAXON_FIELD, TaxonBase.class);
1546         QueryFactory taxonBaseQueryFactory = luceneIndexToolProvider.newQueryFactoryFor(TaxonBase.class);
1547
1548         Builder joinFromQueryBuilder = new Builder();
1549         if(!StringUtils.isEmpty(queryString)){
1550             joinFromQueryBuilder.add(taxonBaseQueryFactory.newTermQuery(queryTermField, queryString), Occur.MUST);
1551         }
1552         joinFromQueryBuilder.add(taxonBaseQueryFactory.newEntityIdsQuery("type.id", edge.getRelationshipTypes()), Occur.MUST);
1553         if(!includeUnpublished){
1554             joinFromQueryBuilder.add(taxonBaseQueryFactory.newBooleanQuery(publishField, true), Occur.MUST);
1555             joinFromQueryBuilder.add(taxonBaseQueryFactory.newBooleanQuery(publishFieldInvers, true), Occur.MUST);
1556         }
1557
1558         Query joinQuery = taxonBaseQueryFactory.newJoinQuery(TaxonRelationship.class, fromField, false, joinFromQueryBuilder.build(), toField, null, ScoreMode.Max);
1559
1560         if(sortFields == null){
1561             sortFields = new  SortField[]{SortField.FIELD_SCORE, new SortField("titleCache__sort", SortField.Type.STRING,  false)};
1562         }
1563         luceneSearch.setSortFields(sortFields);
1564
1565         finalQueryBuilder.add(joinQuery, Occur.MUST);
1566
1567         if(classification != null){
1568             finalQueryBuilder.add(taxonBaseQueryFactory.newEntityIdQuery("taxonNodes.classification.id", classification), Occur.MUST);
1569         }
1570         if(subtree != null){
1571             finalQueryBuilder.add(taxonBaseQueryFactory.newTermQuery("taxonNodes.treeIndex", subtree.treeIndexWc(), true), Occur.MUST);
1572         }
1573
1574         luceneSearch.setQuery(finalQueryBuilder.build());
1575
1576         if(highlightFragments){
1577             luceneSearch.setHighlightFields(taxonBaseQueryFactory.getTextFieldNamesAsArray());
1578         }
1579         return luceneSearch;
1580     }
1581
1582     @Override
1583     public Pager<SearchResult<TaxonBase>> findTaxaAndNamesByFullText(
1584             EnumSet<TaxaAndNamesSearchMode> searchModes, String queryString,
1585             Classification classification, TaxonNode subtree,
1586             Set<NamedArea> namedAreas, Set<PresenceAbsenceTerm> distributionStatus, List<Language> languages,
1587             boolean highlightFragments, Integer pageSize,
1588             Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths)
1589             throws IOException, LuceneParseException, LuceneMultiSearchException {
1590
1591         // FIXME: allow taxonomic ordering
1592         //  hql equivalent:  order by t.name.genusOrUninomial, case when t.name.specificEpithet like '\"%\"' then 1 else 0 end, t.name.specificEpithet, t.name.rank desc, t.name.nameCache";
1593         // this require building a special sort column by a special classBridge
1594         if(highlightFragments){
1595             logger.warn("findTaxaAndNamesByFullText() : fragment highlighting is " +
1596                     "currently not fully supported by this method and thus " +
1597                     "may not work with common names and misapplied names.");
1598         }
1599
1600         // convert sets to lists
1601         List<NamedArea> namedAreaList = null;
1602         List<PresenceAbsenceTerm> distributionStatusList = null;
1603         if(namedAreas != null){
1604             namedAreaList = new ArrayList<>(namedAreas.size());
1605             namedAreaList.addAll(namedAreas);
1606         }
1607         if(distributionStatus != null){
1608             distributionStatusList = new ArrayList<>(distributionStatus.size());
1609             distributionStatusList.addAll(distributionStatus);
1610         }
1611
1612         // set default if parameter is null
1613         if(searchModes == null){
1614             searchModes = EnumSet.of(TaxaAndNamesSearchMode.doTaxa);
1615         }
1616
1617         // set sort order and thus override any sort orders which may have been
1618         // defined by prepare*Search methods
1619         if(orderHints == null){
1620             orderHints = OrderHint.NOMENCLATURAL_SORT_ORDER.asList();
1621         }
1622         SortField[] sortFields = new SortField[orderHints.size()];
1623         int i = 0;
1624         for(OrderHint oh : orderHints){
1625             sortFields[i++] = oh.toSortField();
1626         }
1627 //        SortField[] sortFields = new SortField[]{SortField.FIELD_SCORE, new SortField("id", SortField.STRING, false)};
1628 //        SortField[] sortFields = new SortField[]{new SortField(NomenclaturalSortOrderBrigde.NAME_SORT_FIELD_NAME, SortField.STRING, false)};
1629
1630
1631         boolean addDistributionFilter = namedAreas != null && namedAreas.size() > 0;
1632
1633         List<LuceneSearch> luceneSearches = new ArrayList<>();
1634         Map<CdmBaseType, String> idFieldMap = new HashMap<>();
1635
1636         /*
1637           ======== filtering by distribution , HOWTO ========
1638
1639            - http://www.javaranch.com/journal/2009/02/filtering-a-lucene-search.html
1640            - http://stackoverflow.com/questions/17709256/lucene-solr-using-complex-filters -> QueryWrapperFilter
1641           add Filter to search as http://lucene.apache.org/core/3_6_0/api/all/org/apache/lucene/search/Filter.html
1642           which will be put into a FilteredQuersy  in the end ?
1643
1644
1645           3. how does it work in spatial?
1646           see
1647            - http://www.nsshutdown.com/projects/lucene/whitepaper/locallucene_v2.html
1648            - http://www.infoq.com/articles/LuceneSpatialSupport
1649            - http://www.mhaller.de/archives/156-Spatial-search-with-Lucene.html
1650           ------------------------------------------------------------------------
1651
1652           filter strategies:
1653           A) use a separate distribution filter per index sub-query/search:
1654            - byTaxonSyonym (query TaxaonBase):
1655                use a join area filter (Distribution -> TaxonBase)
1656            - byCommonName (query DescriptionElementBase): use an area filter on
1657                DescriptionElementBase !!! PROBLEM !!!
1658                This cannot work since the distributions are different entities than the
1659                common names and thus these are different lucene documents.
1660            - byMisaplliedNames (join query TaxonRelationship -> TaxonBase):
1661                use a join area filter (Distribution -> TaxonBase)
1662
1663           B) use a common distribution filter for all index sub-query/searches:
1664            - use a common join area filter (Distribution -> TaxonBase)
1665            - also implement the byCommonName as join query (CommonName -> TaxonBase)
1666            PROBLEM in this case: we are losing the fragment highlighting for the
1667            common names, since the returned documents are always TaxonBases
1668         */
1669
1670         /* The QueryFactory for creating filter queries on Distributions should
1671          * The query factory used for the common names query cannot be reused
1672          * for this case, since we want to only record the text fields which are
1673          * actually used in the primary query
1674          */
1675         QueryFactory distributionFilterQueryFactory = luceneIndexToolProvider.newQueryFactoryFor(Distribution.class);
1676
1677         Builder multiIndexByAreaFilterBuilder = new Builder();
1678         boolean includeUnpublished = searchModes.contains(TaxaAndNamesSearchMode.includeUnpublished);
1679
1680         // search for taxa or synonyms
1681         if(searchModes.contains(TaxaAndNamesSearchMode.doTaxa) || searchModes.contains(TaxaAndNamesSearchMode.doSynonyms) ) {
1682             @SuppressWarnings("rawtypes")
1683             Class<? extends TaxonBase> taxonBaseSubclass = TaxonBase.class;
1684             String className = null;
1685             if(searchModes.contains(TaxaAndNamesSearchMode.doTaxa) && !searchModes.contains(TaxaAndNamesSearchMode.doSynonyms)){
1686                 taxonBaseSubclass = Taxon.class;
1687             } else if (!searchModes.contains(TaxaAndNamesSearchMode.doTaxa) && searchModes.contains(TaxaAndNamesSearchMode.doSynonyms)) {
1688                 className = "eu.etaxonomy.cdm.model.taxon.Synonym";
1689             }
1690             luceneSearches.add(prepareFindByFullTextSearch(taxonBaseSubclass,
1691                     queryString, classification, subtree, className,
1692                     includeUnpublished, languages, highlightFragments, sortFields));
1693             idFieldMap.put(CdmBaseType.TAXON, "id");
1694             /* A) does not work!!!!
1695             if(addDistributionFilter){
1696                 // in this case we need a filter which uses a join query
1697                 // to get the TaxonBase documents for the DescriptionElementBase documents
1698                 // which are matching the areas in question
1699                 Query taxonAreaJoinQuery = createByDistributionJoinQuery(
1700                         namedAreaList,
1701                         distributionStatusList,
1702                         distributionFilterQueryFactory
1703                         );
1704                 multiIndexByAreaFilter.add(new QueryWrapperFilter(taxonAreaJoinQuery), Occur.SHOULD);
1705             }
1706             */
1707             if(addDistributionFilter && searchModes.contains(TaxaAndNamesSearchMode.doSynonyms)){
1708                 // add additional area filter for synonyms
1709                 String fromField = "inDescription.taxon.id"; // in DescriptionElementBase index
1710                 String toField = "accTaxon" + AcceptedTaxonBridge.DOC_KEY_ID_SUFFIX; // id in TaxonBase index
1711
1712                 //TODO replace by createByDistributionJoinQuery
1713                 BooleanQuery byDistributionQuery = createByDistributionQuery(namedAreaList, distributionStatusList, distributionFilterQueryFactory);
1714                 Query taxonAreaJoinQuery = distributionFilterQueryFactory.newJoinQuery(Distribution.class, fromField, true, byDistributionQuery, toField, Taxon.class, ScoreMode.None);
1715                 multiIndexByAreaFilterBuilder.add(taxonAreaJoinQuery, Occur.SHOULD);
1716
1717             }
1718         }
1719
1720         // search by CommonTaxonName
1721         if(searchModes.contains(TaxaAndNamesSearchMode.doTaxaByCommonNames)) {
1722             // B)
1723             QueryFactory descriptionElementQueryFactory = luceneIndexToolProvider.newQueryFactoryFor(DescriptionElementBase.class);
1724             Query byCommonNameJoinQuery = descriptionElementQueryFactory.newJoinQuery(
1725                     CommonTaxonName.class,
1726                     "inDescription.taxon.id",
1727                     true,
1728                     QueryFactory.addTypeRestriction(
1729                                 createByDescriptionElementFullTextQuery(queryString, classification, subtree, null, languages, descriptionElementQueryFactory)
1730                                 , CommonTaxonName.class
1731                                 ).build(), "id", null, ScoreMode.Max);
1732             if (logger.isDebugEnabled()){logger.debug("byCommonNameJoinQuery: " + byCommonNameJoinQuery.toString());}
1733             LuceneSearch byCommonNameSearch = new LuceneSearch(luceneIndexToolProvider,
1734                     GroupByTaxonClassBridge.GROUPBY_TAXON_FIELD, Taxon.class);
1735             byCommonNameSearch.setCdmTypRestriction(Taxon.class);
1736             Builder builder = new BooleanQuery.Builder();
1737             builder.add(byCommonNameJoinQuery, Occur.MUST);
1738             if(!includeUnpublished)  {
1739                 QueryFactory taxonBaseQueryFactory = luceneIndexToolProvider.newQueryFactoryFor(TaxonBase.class);
1740                 builder.add(taxonBaseQueryFactory.newBooleanQuery("publish", true), Occur.MUST);
1741             }
1742             byCommonNameSearch.setQuery(builder.build());
1743             byCommonNameSearch.setSortFields(sortFields);
1744
1745             idFieldMap.put(CdmBaseType.TAXON, "id");
1746
1747             luceneSearches.add(byCommonNameSearch);
1748
1749             /* A) does not work!!!!
1750             luceneSearches.add(
1751                     prepareByDescriptionElementFullTextSearch(CommonTaxonName.class,
1752                             queryString, classification, null, languages, highlightFragments)
1753                         );
1754             idFieldMap.put(CdmBaseType.DESCRIPTION_ELEMENT, "inDescription.taxon.id");
1755             if(addDistributionFilter){
1756                 // in this case we are able to use DescriptionElementBase documents
1757                 // which are matching the areas in question directly
1758                 BooleanQuery byDistributionQuery = createByDistributionQuery(
1759                         namedAreaList,
1760                         distributionStatusList,
1761                         distributionFilterQueryFactory
1762                         );
1763                 multiIndexByAreaFilter.add(new QueryWrapperFilter(byDistributionQuery), Occur.SHOULD);
1764             } */
1765         }
1766
1767
1768         // search by misapplied names
1769         //TODO merge with pro parte synonym search once #7487 is fixed
1770         if(searchModes.contains(TaxaAndNamesSearchMode.doMisappliedNames) /*|| searchModes.contains(TaxaAndNamesSearchMode.doSynonyms) */) {
1771             // NOTE:
1772             // prepareFindByTaxonRelationFullTextSearch() is making use of JoinUtil.createJoinQuery()
1773             // which allows doing query time joins
1774             // finds the misapplied name (Taxon B) which is an misapplication for
1775             // a related Taxon A.
1776             //
1777             Set<TaxonRelationshipType> relTypes = new HashSet<>();
1778             if (searchModes.contains(TaxaAndNamesSearchMode.doMisappliedNames)){
1779                 relTypes.addAll(TaxonRelationshipType.allMisappliedNameTypes());
1780             }
1781 //            if (searchModes.contains(TaxaAndNamesSearchMode.doSynonyms)){
1782 //                relTypes.addAll(TaxonRelationshipType.allSynonymTypes());
1783 //            }
1784
1785             luceneSearches.add(prepareFindByTaxonRelationFullTextSearch(
1786                     new TaxonRelationshipEdge(relTypes, Direction.relatedTo),
1787                     queryString, classification, subtree, includeUnpublished, languages, highlightFragments, sortFields));
1788             idFieldMap.put(CdmBaseType.TAXON, "id");
1789
1790             if(addDistributionFilter){
1791                 String fromField = "inDescription.taxon.id"; // in DescriptionElementBase index
1792
1793                 /*
1794                  * Here I was facing a weird and nasty bug which took me bugging be really for hours until I found this solution.
1795                  * Maybe this is a bug in java itself.
1796                  *
1797                  * When the string toField is constructed by using the expression TaxonRelationshipType.MISAPPLIED_NAME_FOR().getUuid().toString()
1798                  * directly:
1799                  *
1800                  *    String toField = "relation." + TaxonRelationshipType.MISAPPLIED_NAME_FOR().getUuid().toString() +".to.id";
1801                  *
1802                  * The byDistributionQuery fails, however when the uuid is first stored in another string variable the query
1803                  * will execute as expected:
1804                  *
1805                  *    String misappliedNameForUuid = TaxonRelationshipType.MISAPPLIED_NAME_FOR().getUuid().toString();
1806                  *    String toField = "relation." + misappliedNameForUuid +".to.id";
1807                  *
1808                  * Comparing both strings by the String.equals method returns true, so both String are identical.
1809                  *
1810                  * The bug occurs when running eu.etaxonomy.cdm.api.service.TaxonServiceSearchTest in eclipse and in maven and seems to to be
1811                  * dependent from a specific jvm (openjdk6  6b27-1.12.6-1ubuntu0.13.04.2, openjdk7 7u25-2.3.10-1ubuntu0.13.04.2,  oracle jdk1.7.0_25 tested)
1812                  * The bug is persistent after a reboot of the development computer.
1813                  */
1814 //                String misappliedNameForUuid = TaxonRelationshipType.MISAPPLIED_NAME_FOR().getUuid().toString();
1815 //                String toField = "relation." + misappliedNameForUuid +".to.id";
1816                 String toField = "relation.1ed87175-59dd-437e-959e-0d71583d8417.to.id";
1817 //                System.out.println("relation.1ed87175-59dd-437e-959e-0d71583d8417.to.id".equals("relation." + misappliedNameForUuid +".to.id") ? " > identical" : " > different");
1818 //                System.out.println("relation.1ed87175-59dd-437e-959e-0d71583d8417.to.id".equals("relation." + TaxonRelationshipType.MISAPPLIED_NAME_FOR().getUuid().toString() +".to.id") ? " > identical" : " > different");
1819
1820                 //TODO replace by createByDistributionJoinQuery
1821                 BooleanQuery byDistributionQuery = createByDistributionQuery(namedAreaList, distributionStatusList, distributionFilterQueryFactory);
1822                 Query taxonAreaJoinQuery = distributionFilterQueryFactory.newJoinQuery(Distribution.class,
1823                         fromField, true, byDistributionQuery, toField, null, ScoreMode.None);
1824
1825 //                debug code for bug described above
1826                 //does not compile anymore since changing from lucene 3.6.2 to lucene 4.10+
1827 //                DocIdSet filterMatchSet = filter.getDocIdSet(luceneIndexToolProvider.getIndexReaderFor(Taxon.class));
1828 //                System.err.println(DocIdBitSetPrinter.docsAsString(filterMatchSet, 100));
1829
1830                 multiIndexByAreaFilterBuilder.add(taxonAreaJoinQuery, Occur.SHOULD);
1831             }
1832         }
1833
1834
1835         // search by pro parte synonyms
1836         if(searchModes.contains(TaxaAndNamesSearchMode.doSynonyms)) {
1837             //TODO merge with misapplied name search once #7487 is fixed
1838             Set<TaxonRelationshipType> relTypes = new HashSet<>();
1839             relTypes.addAll(TaxonRelationshipType.allSynonymTypes());
1840
1841             luceneSearches.add(prepareFindByTaxonRelationFullTextSearch(
1842                     new TaxonRelationshipEdge(relTypes, Direction.relatedTo),
1843                     queryString, classification, subtree, includeUnpublished, languages, highlightFragments, sortFields));
1844             idFieldMap.put(CdmBaseType.TAXON, "id");
1845
1846             if(addDistributionFilter){
1847                 String fromField = "inDescription.taxon.id"; // in DescriptionElementBase index
1848                 String toField = "relation.8a896603-0fa3-44c6-9cd7-df2d8792e577.to.id";
1849                 BooleanQuery byDistributionQuery = createByDistributionQuery(namedAreaList, distributionStatusList, distributionFilterQueryFactory);
1850                 Query taxonAreaJoinQuery = distributionFilterQueryFactory.newJoinQuery(Distribution.class,
1851                         fromField, true, byDistributionQuery, toField, null, ScoreMode.None);
1852                 multiIndexByAreaFilterBuilder.add(taxonAreaJoinQuery, Occur.SHOULD);
1853             }
1854         }//end pro parte synonyms
1855
1856
1857
1858         LuceneMultiSearch multiSearch = new LuceneMultiSearch(luceneIndexToolProvider,
1859                 luceneSearches.toArray(new LuceneSearch[luceneSearches.size()]));
1860
1861
1862         if(addDistributionFilter){
1863
1864             // B)
1865             // in this case we need a filter which uses a join query
1866             // to get the TaxonBase documents for the DescriptionElementBase documents
1867             // which are matching the areas in question
1868             //
1869             // for doTaxa, doByCommonName
1870             Query taxonAreaJoinQuery = createByDistributionJoinQuery(
1871                     namedAreaList,
1872                     distributionStatusList,
1873                     distributionFilterQueryFactory,
1874                     Taxon.class, true
1875                     );
1876             multiIndexByAreaFilterBuilder.add(taxonAreaJoinQuery, Occur.SHOULD);
1877         }
1878
1879         if (addDistributionFilter){
1880             multiSearch.setFilter(multiIndexByAreaFilterBuilder.build());
1881         }
1882
1883
1884         // --- execute search
1885         TopGroups<BytesRef> topDocsResultSet;
1886         try {
1887             topDocsResultSet = multiSearch.executeSearch(pageSize, pageNumber);
1888         } catch (ParseException e) {
1889             LuceneParseException luceneParseException = new LuceneParseException(e.getMessage());
1890             luceneParseException.setStackTrace(e.getStackTrace());
1891             throw luceneParseException;
1892         }
1893
1894         // --- initialize taxa, highlight matches ....
1895         ISearchResultBuilder searchResultBuilder = new SearchResultBuilder(multiSearch, multiSearch.getQuery());
1896
1897
1898         List<SearchResult<TaxonBase>> searchResults = searchResultBuilder.createResultSet(
1899                 topDocsResultSet, multiSearch.getHighlightFields(), dao, idFieldMap, propertyPaths);
1900
1901         long totalHits = (topDocsResultSet != null) ? Long.valueOf(topDocsResultSet.totalGroupCount) : 0;
1902         return new DefaultPagerImpl<>(pageNumber, totalHits, pageSize, searchResults);
1903     }
1904
1905     /**
1906      * @param namedAreaList at least one area must be in the list
1907      * @param distributionStatusList optional
1908      * @param toType toType
1909      *      Optional parameter. Only used for debugging to print the toType documents
1910      * @param asFilter TODO
1911      * @return
1912      * @throws IOException
1913      */
1914     protected Query createByDistributionJoinQuery(
1915             List<NamedArea> namedAreaList,
1916             List<PresenceAbsenceTerm> distributionStatusList,
1917             QueryFactory queryFactory, Class<? extends CdmBase> toType, boolean asFilter
1918             ) throws IOException {
1919
1920         String fromField = "inDescription.taxon.id"; // in DescriptionElementBase index
1921         String toField = "id"; // id in toType usually this is the TaxonBase index
1922
1923         BooleanQuery byDistributionQuery = createByDistributionQuery(namedAreaList, distributionStatusList, queryFactory);
1924
1925         ScoreMode scoreMode = asFilter ?  ScoreMode.None : ScoreMode.Max;
1926
1927         Query taxonAreaJoinQuery = queryFactory.newJoinQuery(Distribution.class, fromField, false, byDistributionQuery, toField, toType, scoreMode);
1928
1929         return taxonAreaJoinQuery;
1930     }
1931
1932     /**
1933      * @param namedAreaList
1934      * @param distributionStatusList
1935      * @param queryFactory
1936      * @return
1937      */
1938     private BooleanQuery createByDistributionQuery(List<NamedArea> namedAreaList,
1939             List<PresenceAbsenceTerm> distributionStatusList, QueryFactory queryFactory) {
1940         Builder areaQueryBuilder = new Builder();
1941         // area field from Distribution
1942         areaQueryBuilder.add(queryFactory.newEntityIdsQuery("area.id", namedAreaList), Occur.MUST);
1943
1944         // status field from Distribution
1945         if(distributionStatusList != null && distributionStatusList.size() > 0){
1946             areaQueryBuilder.add(queryFactory.newEntityIdsQuery("status.id", distributionStatusList), Occur.MUST);
1947         }
1948
1949         BooleanQuery areaQuery = areaQueryBuilder.build();
1950         logger.debug("createByDistributionQuery() query: " + areaQuery.toString());
1951         return areaQuery;
1952     }
1953
1954     /**
1955      * This method has been primarily created for testing the area join query but might
1956      * also be useful in other situations
1957      *
1958      * @param namedAreaList
1959      * @param distributionStatusList
1960      * @param classification
1961      * @param highlightFragments
1962      * @return
1963      * @throws IOException
1964      */
1965     protected LuceneSearch prepareByDistributionSearch(
1966             List<NamedArea> namedAreaList, List<PresenceAbsenceTerm> distributionStatusList,
1967             Classification classification) throws IOException {
1968
1969         Builder finalQueryBuilder = new Builder();
1970
1971         LuceneSearch luceneSearch = new LuceneSearch(luceneIndexToolProvider, GroupByTaxonClassBridge.GROUPBY_TAXON_FIELD, Taxon.class);
1972
1973         // FIXME is this query factory using the wrong type?
1974         QueryFactory taxonQueryFactory = luceneIndexToolProvider.newQueryFactoryFor(Taxon.class);
1975
1976         SortField[] sortFields = new  SortField[]{SortField.FIELD_SCORE, new SortField("titleCache__sort", SortField.Type.STRING, false)};
1977         luceneSearch.setSortFields(sortFields);
1978
1979
1980         Query byAreaQuery = createByDistributionJoinQuery(namedAreaList, distributionStatusList, taxonQueryFactory, null, false);
1981
1982         finalQueryBuilder.add(byAreaQuery, Occur.MUST);
1983
1984         if(classification != null){
1985             finalQueryBuilder.add(taxonQueryFactory.newEntityIdQuery("taxonNodes.classification.id", classification), Occur.MUST);
1986         }
1987         BooleanQuery finalQuery = finalQueryBuilder.build();
1988         logger.info("prepareByAreaSearch() query: " + finalQuery.toString());
1989         luceneSearch.setQuery(finalQuery);
1990
1991         return luceneSearch;
1992     }
1993
1994     @Override
1995     public Pager<SearchResult<TaxonBase>> findByDescriptionElementFullText(
1996             Class<? extends DescriptionElementBase> clazz, String queryString,
1997             Classification classification, TaxonNode subtree, List<Feature> features, List<Language> languages,
1998             boolean highlightFragments, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) throws IOException, LuceneParseException {
1999
2000
2001         LuceneSearch luceneSearch = prepareByDescriptionElementFullTextSearch(clazz, queryString, classification, subtree, features, languages, highlightFragments);
2002
2003         // --- execute search
2004         TopGroups<BytesRef> topDocsResultSet;
2005         try {
2006             topDocsResultSet = luceneSearch.executeSearch(pageSize, pageNumber);
2007         } catch (ParseException e) {
2008             LuceneParseException luceneParseException = new LuceneParseException(e.getMessage());
2009             luceneParseException.setStackTrace(e.getStackTrace());
2010             throw luceneParseException;
2011         }
2012
2013         Map<CdmBaseType, String> idFieldMap = new HashMap<>();
2014         idFieldMap.put(CdmBaseType.DESCRIPTION_ELEMENT, "inDescription.taxon.id");
2015
2016         // --- initialize taxa, highlight matches ....
2017         ISearchResultBuilder searchResultBuilder = new SearchResultBuilder(luceneSearch, luceneSearch.getQuery());
2018         @SuppressWarnings("rawtypes")
2019         List<SearchResult<TaxonBase>> searchResults = searchResultBuilder.createResultSet(
2020                 topDocsResultSet, luceneSearch.getHighlightFields(), dao, idFieldMap, propertyPaths);
2021
2022         int totalHits = topDocsResultSet != null ? topDocsResultSet.totalGroupCount : 0;
2023         return new DefaultPagerImpl<>(pageNumber, Long.valueOf(totalHits), pageSize, searchResults);
2024
2025     }
2026
2027
2028     @Override
2029     public Pager<SearchResult<TaxonBase>> findByEverythingFullText(String queryString,
2030             Classification classification, TaxonNode subtree, boolean includeUnpublished, List<Language> languages, boolean highlightFragments,
2031             Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) throws IOException, LuceneParseException, LuceneMultiSearchException {
2032
2033         LuceneSearch luceneSearchByDescriptionElement = prepareByDescriptionElementFullTextSearch(null, queryString,
2034                 classification, subtree,
2035                 null, languages, highlightFragments);
2036         LuceneSearch luceneSearchByTaxonBase = prepareFindByFullTextSearch(null, queryString, classification, subtree, null,
2037                 includeUnpublished, languages, highlightFragments, null);
2038
2039         LuceneMultiSearch multiSearch = new LuceneMultiSearch(luceneIndexToolProvider, luceneSearchByDescriptionElement, luceneSearchByTaxonBase);
2040
2041         // --- execute search
2042         TopGroups<BytesRef> topDocsResultSet;
2043         try {
2044             topDocsResultSet = multiSearch.executeSearch(pageSize, pageNumber);
2045         } catch (ParseException e) {
2046             LuceneParseException luceneParseException = new LuceneParseException(e.getMessage());
2047             luceneParseException.setStackTrace(e.getStackTrace());
2048             throw luceneParseException;
2049         }
2050
2051         // --- initialize taxa, highlight matches ....
2052         ISearchResultBuilder searchResultBuilder = new SearchResultBuilder(multiSearch, multiSearch.getQuery());
2053
2054         Map<CdmBaseType, String> idFieldMap = new HashMap<>();
2055         idFieldMap.put(CdmBaseType.TAXON, "id");
2056         idFieldMap.put(CdmBaseType.DESCRIPTION_ELEMENT, "inDescription.taxon.id");
2057
2058         List<SearchResult<TaxonBase>> searchResults = searchResultBuilder.createResultSet(
2059                 topDocsResultSet, multiSearch.getHighlightFields(), dao, idFieldMap, propertyPaths);
2060
2061         int totalHits = topDocsResultSet != null ? topDocsResultSet.totalGroupCount : 0;
2062         return new DefaultPagerImpl<>(pageNumber, Long.valueOf(totalHits), pageSize, searchResults);
2063
2064     }
2065
2066
2067     /**
2068      * @param clazz
2069      * @param queryString
2070      * @param classification
2071      * @param features
2072      * @param languages
2073      * @param highlightFragments
2074      * @param directorySelectClass
2075      * @return
2076      */
2077     protected LuceneSearch prepareByDescriptionElementFullTextSearch(Class<? extends CdmBase> clazz,
2078             String queryString, Classification classification, TaxonNode subtree, List<Feature> features,
2079             List<Language> languages, boolean highlightFragments) {
2080
2081         LuceneSearch luceneSearch = new LuceneSearch(luceneIndexToolProvider, GroupByTaxonClassBridge.GROUPBY_TAXON_FIELD, DescriptionElementBase.class);
2082         QueryFactory descriptionElementQueryFactory = luceneIndexToolProvider.newQueryFactoryFor(DescriptionElementBase.class);
2083
2084         SortField[] sortFields = new  SortField[]{SortField.FIELD_SCORE, new SortField("inDescription.taxon.titleCache__sort", SortField.Type.STRING, false)};
2085
2086         BooleanQuery finalQuery = createByDescriptionElementFullTextQuery(queryString, classification, subtree, features,
2087                 languages, descriptionElementQueryFactory);
2088
2089         luceneSearch.setSortFields(sortFields);
2090         luceneSearch.setCdmTypRestriction(clazz);
2091         luceneSearch.setQuery(finalQuery);
2092         if(highlightFragments){
2093             luceneSearch.setHighlightFields(descriptionElementQueryFactory.getTextFieldNamesAsArray());
2094         }
2095
2096         return luceneSearch;
2097     }
2098
2099     /**
2100      * @param queryString
2101      * @param classification
2102      * @param features
2103      * @param languages
2104      * @param descriptionElementQueryFactory
2105      * @return
2106      */
2107     private BooleanQuery createByDescriptionElementFullTextQuery(String queryString,
2108             Classification classification, TaxonNode subtree, List<Feature> features,
2109             List<Language> languages, QueryFactory descriptionElementQueryFactory) {
2110
2111         Builder finalQueryBuilder = new Builder();
2112         Builder textQueryBuilder = new Builder();
2113
2114         if(!StringUtils.isEmpty(queryString)){
2115
2116             textQueryBuilder.add(descriptionElementQueryFactory.newTermQuery("titleCache", queryString), Occur.SHOULD);
2117
2118             // common name
2119             Builder nameQueryBuilder = new Builder();
2120             if(languages == null || languages.size() == 0){
2121                 nameQueryBuilder.add(descriptionElementQueryFactory.newTermQuery("name", queryString), Occur.MUST);
2122             } else {
2123                 Builder languageSubQueryBuilder = new Builder();
2124                 for(Language lang : languages){
2125                     languageSubQueryBuilder.add(descriptionElementQueryFactory.newTermQuery("language.uuid",  lang.getUuid().toString(), false), Occur.SHOULD);
2126                 }
2127                 nameQueryBuilder.add(descriptionElementQueryFactory.newTermQuery("name", queryString), Occur.MUST);
2128                 nameQueryBuilder.add(languageSubQueryBuilder.build(), Occur.MUST);
2129             }
2130             textQueryBuilder.add(nameQueryBuilder.build(), Occur.SHOULD);
2131
2132
2133             // text field from TextData
2134             textQueryBuilder.add(descriptionElementQueryFactory.newMultilanguageTextQuery("text", queryString, languages), Occur.SHOULD);
2135
2136             // --- TermBase fields - by representation ----
2137             // state field from CategoricalData
2138             textQueryBuilder.add(descriptionElementQueryFactory.newDefinedTermQuery("stateData.state", queryString, languages), Occur.SHOULD);
2139
2140             // state field from CategoricalData
2141             textQueryBuilder.add(descriptionElementQueryFactory.newDefinedTermQuery("stateData.modifyingText", queryString, languages), Occur.SHOULD);
2142
2143             // area field from Distribution
2144             textQueryBuilder.add(descriptionElementQueryFactory.newDefinedTermQuery("area", queryString, languages), Occur.SHOULD);
2145
2146             // status field from Distribution
2147             textQueryBuilder.add(descriptionElementQueryFactory.newDefinedTermQuery("status", queryString, languages), Occur.SHOULD);
2148
2149             finalQueryBuilder.add(textQueryBuilder.build(), Occur.MUST);
2150
2151         }
2152         // --- classification ----
2153
2154
2155         if(classification != null){
2156             finalQueryBuilder.add(descriptionElementQueryFactory.newEntityIdQuery("inDescription.taxon.taxonNodes.classification.id", classification), Occur.MUST);
2157         }
2158         if(subtree != null){
2159             finalQueryBuilder.add(descriptionElementQueryFactory.newTermQuery("inDescription.taxon.taxonNodes.treeIndex", subtree.treeIndexWc(), true), Occur.MUST);
2160         }
2161
2162         // --- IdentifieableEntity fields - by uuid
2163         if(features != null && features.size() > 0 ){
2164             finalQueryBuilder.add(descriptionElementQueryFactory.newEntityUuidsQuery("feature.uuid", features), Occur.MUST);
2165         }
2166
2167         // the description must be associated with a taxon
2168         finalQueryBuilder.add(descriptionElementQueryFactory.newIsNotNullQuery("inDescription.taxon.id"), Occur.MUST);
2169
2170         BooleanQuery finalQuery = finalQueryBuilder.build();
2171         logger.info("prepareByDescriptionElementFullTextSearch() query: " + finalQuery.toString());
2172         return finalQuery;
2173     }
2174
2175     @Override
2176     public List<Synonym> createInferredSynonyms(Taxon taxon, Classification classification, SynonymType type, boolean doWithMisappliedNames){
2177
2178
2179         List <Synonym> inferredSynonyms = new ArrayList<>();
2180         List<Synonym> inferredSynonymsToBeRemoved = new ArrayList<>();
2181
2182         Map <UUID, IZoologicalName> zooHashMap = new HashMap<>();
2183         boolean includeUnpublished = INCLUDE_UNPUBLISHED;
2184
2185         UUID nameUuid= taxon.getName().getUuid();
2186         IZoologicalName taxonName = getZoologicalName(nameUuid, zooHashMap);
2187         String epithetOfTaxon = null;
2188         String infragenericEpithetOfTaxon = null;
2189         String infraspecificEpithetOfTaxon = null;
2190         if (taxonName.isSpecies()){
2191              epithetOfTaxon= taxonName.getSpecificEpithet();
2192         } else if (taxonName.isInfraGeneric()){
2193             infragenericEpithetOfTaxon = taxonName.getInfraGenericEpithet();
2194         } else if (taxonName.isInfraSpecific()){
2195             infraspecificEpithetOfTaxon = taxonName.getInfraSpecificEpithet();
2196         }
2197         String genusOfTaxon = taxonName.getGenusOrUninomial();
2198         Set<TaxonNode> nodes = taxon.getTaxonNodes();
2199         List<String> taxonNames = new ArrayList<>();
2200
2201         for (TaxonNode node: nodes){
2202            // Map<String, String> synonymsGenus = new HashMap<>(); // Changed this to be able to store the idInSource to a genusName
2203            // List<String> synonymsEpithet = new ArrayList<>();
2204
2205             if (node.getClassification().equals(classification)){
2206                 if (!node.isTopmostNode()){
2207                     TaxonNode parent = node.getParent();
2208                     parent = CdmBase.deproxy(parent);
2209                     TaxonName parentName =  parent.getTaxon().getName();
2210                     IZoologicalName zooParentName = CdmBase.deproxy(parentName);
2211                     Taxon parentTaxon = CdmBase.deproxy(parent.getTaxon());
2212
2213                     //create inferred synonyms for species, subspecies
2214                     if ((parentName.isGenus() || parentName.isSpecies() || parentName.getRank().equals(Rank.SUBGENUS())) ){
2215
2216                         Synonym inferredEpithet = null;
2217                         Synonym inferredGenus = null;
2218                         Synonym potentialCombination = null;
2219
2220                         List<String> propertyPaths = new ArrayList<>();
2221                         propertyPaths.add("synonym");
2222                         propertyPaths.add("synonym.name");
2223                         List<OrderHint> orderHintsSynonyms = new ArrayList<>();
2224                         orderHintsSynonyms.add(new OrderHint("titleCache", SortOrder.ASCENDING));
2225
2226                         List<Synonym> synonyMsOfParent = dao.getSynonyms(parentTaxon, SynonymType.HETEROTYPIC_SYNONYM_OF(), null, null,orderHintsSynonyms,propertyPaths);
2227                         List<Synonym> synonymsOfTaxon= dao.getSynonyms(taxon, SynonymType.HETEROTYPIC_SYNONYM_OF(),
2228                                 null, null,orderHintsSynonyms,propertyPaths);
2229
2230                         List<TaxonRelationship> taxonRelListParent = new ArrayList<>();
2231                         List<TaxonRelationship> taxonRelListTaxon = new ArrayList<>();
2232                         if (doWithMisappliedNames){
2233                             List<OrderHint> orderHintsMisapplied = new ArrayList<>();
2234                             orderHintsMisapplied.add(new OrderHint("relatedFrom.titleCache", SortOrder.ASCENDING));
2235                             taxonRelListParent = dao.getTaxonRelationships(parentTaxon, TaxonRelationshipType.MISAPPLIED_NAME_FOR(),
2236                                     includeUnpublished, null, null, orderHintsMisapplied, propertyPaths, Direction.relatedTo);
2237                             taxonRelListTaxon = dao.getTaxonRelationships(taxon, TaxonRelationshipType.MISAPPLIED_NAME_FOR(),
2238                                     includeUnpublished, null, null, orderHintsMisapplied, propertyPaths, Direction.relatedTo);
2239                         }
2240
2241                         if (type.equals(SynonymType.INFERRED_EPITHET_OF())){
2242                             for (Synonym synonymRelationOfParent:synonyMsOfParent){
2243
2244                                 inferredEpithet = createInferredEpithets(taxon,
2245                                         zooHashMap, taxonName, epithetOfTaxon,
2246                                         infragenericEpithetOfTaxon,
2247                                         infraspecificEpithetOfTaxon,
2248                                         taxonNames, parentName,
2249                                         synonymRelationOfParent);
2250
2251                                 inferredSynonyms.add(inferredEpithet);
2252                                 zooHashMap.put(inferredEpithet.getName().getUuid(), inferredEpithet.getName());
2253                                 taxonNames.add(inferredEpithet.getName().getNameCache());
2254                             }
2255
2256                             if (doWithMisappliedNames){
2257
2258                                 for (TaxonRelationship taxonRelationship: taxonRelListParent){
2259                                      Taxon misappliedName = taxonRelationship.getFromTaxon();
2260
2261                                      inferredEpithet = createInferredEpithets(taxon,
2262                                              zooHashMap, taxonName, epithetOfTaxon,
2263                                              infragenericEpithetOfTaxon,
2264                                              infraspecificEpithetOfTaxon,
2265                                              taxonNames, parentName,
2266                                              misappliedName);
2267
2268                                     inferredSynonyms.add(inferredEpithet);
2269                                     zooHashMap.put(inferredEpithet.getName().getUuid(), inferredEpithet.getName());
2270                                     taxonNames.add(inferredEpithet.getName().getNameCache());
2271                                 }
2272                             }
2273
2274                             if (!taxonNames.isEmpty()){
2275                             List<String> synNotInCDM = dao.taxaByNameNotInDB(taxonNames);
2276                             IZoologicalName name;
2277                             if (!synNotInCDM.isEmpty()){
2278                                 inferredSynonymsToBeRemoved.clear();
2279
2280                                 for (Synonym syn :inferredSynonyms){
2281                                     name = getZoologicalName(syn.getName().getUuid(), zooHashMap);
2282                                     if (!synNotInCDM.contains(name.getNameCache())){
2283                                         inferredSynonymsToBeRemoved.add(syn);
2284                                     }
2285                                 }
2286
2287                                 // Remove identified Synonyms from inferredSynonyms
2288                                 for (Synonym synonym : inferredSynonymsToBeRemoved) {
2289                                     inferredSynonyms.remove(synonym);
2290                                 }
2291                             }
2292                         }
2293
2294                     }else if (type.equals(SynonymType.INFERRED_GENUS_OF())){
2295
2296                         for (Synonym synonymRelationOfTaxon:synonymsOfTaxon){
2297
2298                             inferredGenus = createInferredGenus(taxon,
2299                                     zooHashMap, taxonName, epithetOfTaxon,
2300                                     genusOfTaxon, taxonNames, zooParentName, synonymRelationOfTaxon);
2301
2302                             inferredSynonyms.add(inferredGenus);
2303                             zooHashMap.put(inferredGenus.getName().getUuid(), inferredGenus.getName());
2304                             taxonNames.add(inferredGenus.getName().getNameCache());
2305                         }
2306
2307                         if (doWithMisappliedNames){
2308
2309                             for (TaxonRelationship taxonRelationship: taxonRelListTaxon){
2310                                 Taxon misappliedName = taxonRelationship.getFromTaxon();
2311                                 inferredGenus = createInferredGenus(taxon, zooHashMap, taxonName, infraspecificEpithetOfTaxon, genusOfTaxon, taxonNames, zooParentName,  misappliedName);
2312
2313                                 inferredSynonyms.add(inferredGenus);
2314                                 zooHashMap.put(inferredGenus.getName().getUuid(), inferredGenus.getName());
2315                                  taxonNames.add(inferredGenus.getName().getNameCache());
2316                             }
2317                         }
2318
2319
2320                         if (!taxonNames.isEmpty()){
2321                             List<String> synNotInCDM = dao.taxaByNameNotInDB(taxonNames);
2322                             IZoologicalName name;
2323                             if (!synNotInCDM.isEmpty()){
2324                                 inferredSynonymsToBeRemoved.clear();
2325
2326                                 for (Synonym syn :inferredSynonyms){
2327                                     name = getZoologicalName(syn.getName().getUuid(), zooHashMap);
2328                                     if (!synNotInCDM.contains(name.getNameCache())){
2329                                         inferredSynonymsToBeRemoved.add(syn);
2330                                     }
2331                                 }
2332
2333                                 // Remove identified Synonyms from inferredSynonyms
2334                                 for (Synonym synonym : inferredSynonymsToBeRemoved) {
2335                                     inferredSynonyms.remove(synonym);
2336                                 }
2337                             }
2338                         }
2339
2340                     }else if (type.equals(SynonymType.POTENTIAL_COMBINATION_OF())){
2341
2342                         Reference sourceReference = null; // TODO: Determination of sourceReference is redundant
2343                         //for all synonyms of the parent...
2344                         for (Synonym synonymRelationOfParent:synonyMsOfParent){
2345                             TaxonName synName;
2346                             HibernateProxyHelper.deproxy(synonymRelationOfParent);
2347
2348                             synName = synonymRelationOfParent.getName();
2349
2350                             // Set the sourceReference
2351                             sourceReference = synonymRelationOfParent.getSec();
2352
2353                             // Determine the idInSource
2354                             String idInSourceParent = getIdInSource(synonymRelationOfParent);
2355
2356                             IZoologicalName parentSynZooName = getZoologicalName(synName.getUuid(), zooHashMap);
2357                             String synParentGenus = parentSynZooName.getGenusOrUninomial();
2358                             String synParentInfragenericName = null;
2359                             String synParentSpecificEpithet = null;
2360
2361                             if (parentSynZooName.isInfraGeneric()){
2362                                 synParentInfragenericName = parentSynZooName.getInfraGenericEpithet();
2363                             }
2364                             if (parentSynZooName.isSpecies()){
2365                                 synParentSpecificEpithet = parentSynZooName.getSpecificEpithet();
2366                             }
2367
2368                            /* if (synGenusName != null && !synonymsGenus.containsKey(synGenusName)){
2369                                 synonymsGenus.put(synGenusName, idInSource);
2370                             }*/
2371
2372                             //for all synonyms of the taxon
2373
2374                             for (Synonym synonymRelationOfTaxon:synonymsOfTaxon){
2375
2376                                 IZoologicalName zooSynName = getZoologicalName(synonymRelationOfTaxon.getName().getUuid(), zooHashMap);
2377                                 potentialCombination = createPotentialCombination(idInSourceParent, parentSynZooName, zooSynName,
2378                                         synParentGenus,
2379                                         synParentInfragenericName,
2380                                         synParentSpecificEpithet, synonymRelationOfTaxon, zooHashMap);
2381
2382                                 taxon.addSynonym(potentialCombination, SynonymType.POTENTIAL_COMBINATION_OF());
2383                                 inferredSynonyms.add(potentialCombination);
2384                                 zooHashMap.put(potentialCombination.getName().getUuid(), potentialCombination.getName());
2385                                  taxonNames.add(potentialCombination.getName().getNameCache());
2386
2387                             }
2388
2389                         }
2390
2391                         if (doWithMisappliedNames){
2392
2393                             for (TaxonRelationship parentRelationship: taxonRelListParent){
2394
2395                                 TaxonName misappliedParentName;
2396
2397                                 Taxon misappliedParent = parentRelationship.getFromTaxon();
2398                                 misappliedParentName = misappliedParent.getName();
2399
2400                                 HibernateProxyHelper.deproxy(misappliedParent);
2401
2402                                 // Set the sourceReference
2403                                 sourceReference = misappliedParent.getSec();
2404
2405                                 // Determine the idInSource
2406                                 String idInSourceParent = getIdInSource(misappliedParent);
2407
2408                                 IZoologicalName parentSynZooName = getZoologicalName(misappliedParentName.getUuid(), zooHashMap);
2409                                 String synParentGenus = parentSynZooName.getGenusOrUninomial();
2410                                 String synParentInfragenericName = null;
2411                                 String synParentSpecificEpithet = null;
2412
2413                                 if (parentSynZooName.isInfraGeneric()){
2414                                     synParentInfragenericName = parentSynZooName.getInfraGenericEpithet();
2415                                 }
2416                                 if (parentSynZooName.isSpecies()){
2417                                     synParentSpecificEpithet = parentSynZooName.getSpecificEpithet();
2418                                 }
2419
2420
2421                                 for (TaxonRelationship taxonRelationship: taxonRelListTaxon){
2422                                     Taxon misappliedName = taxonRelationship.getFromTaxon();
2423                                     IZoologicalName zooMisappliedName = getZoologicalName(misappliedName.getName().getUuid(), zooHashMap);
2424                                     potentialCombination = createPotentialCombination(
2425                                             idInSourceParent, parentSynZooName, zooMisappliedName,
2426                                             synParentGenus,
2427                                             synParentInfragenericName,
2428                                             synParentSpecificEpithet, misappliedName, zooHashMap);
2429
2430
2431                                     taxon.addSynonym(potentialCombination, SynonymType.POTENTIAL_COMBINATION_OF());
2432                                     inferredSynonyms.add(potentialCombination);
2433                                     zooHashMap.put(potentialCombination.getName().getUuid(), potentialCombination.getName());
2434                                      taxonNames.add(potentialCombination.getName().getNameCache());
2435                                 }
2436                             }
2437                         }
2438
2439                         if (!taxonNames.isEmpty()){
2440                             List<String> synNotInCDM = dao.taxaByNameNotInDB(taxonNames);
2441                             IZoologicalName name;
2442                             if (!synNotInCDM.isEmpty()){
2443                                 inferredSynonymsToBeRemoved.clear();
2444                                 for (Synonym syn :inferredSynonyms){
2445                                     try{
2446                                         name = syn.getName();
2447                                     }catch (ClassCastException e){
2448                                         name = getZoologicalName(syn.getName().getUuid(), zooHashMap);
2449                                     }
2450                                     if (!synNotInCDM.contains(name.getNameCache())){
2451                                         inferredSynonymsToBeRemoved.add(syn);
2452                                     }
2453                                  }
2454                                 // Remove identified Synonyms from inferredSynonyms
2455                                 for (Synonym synonym : inferredSynonymsToBeRemoved) {
2456                                     inferredSynonyms.remove(synonym);
2457                                 }
2458                             }
2459                          }
2460                         }
2461                     }else {
2462                         logger.info("The synonym type is not defined.");
2463                         return inferredSynonyms;
2464                     }
2465                 }
2466             }
2467
2468         }
2469
2470         return inferredSynonyms;
2471     }
2472
2473     private Synonym createPotentialCombination(String idInSourceParent,
2474             IZoologicalName parentSynZooName,   IZoologicalName zooSynName, String synParentGenus,
2475             String synParentInfragenericName, String synParentSpecificEpithet,
2476             TaxonBase<?> syn, Map<UUID, IZoologicalName> zooHashMap) {
2477         Synonym potentialCombination;
2478         Reference sourceReference;
2479         IZoologicalName inferredSynName;
2480         HibernateProxyHelper.deproxy(syn);
2481
2482         // Set sourceReference
2483         sourceReference = syn.getSec();
2484         if (sourceReference == null){
2485             logger.warn("The synonym has no sec reference because it is a misapplied name! Take the sec reference of taxon");
2486             //TODO:Remove
2487             if (!parentSynZooName.getTaxa().isEmpty()){
2488                 TaxonBase<?> taxon = parentSynZooName.getTaxa().iterator().next();
2489
2490                 sourceReference = taxon.getSec();
2491             }
2492         }
2493         String synTaxonSpecificEpithet = zooSynName.getSpecificEpithet();
2494
2495         String synTaxonInfraSpecificName= null;
2496
2497         if (parentSynZooName.isSpecies()){
2498             synTaxonInfraSpecificName = zooSynName.getInfraSpecificEpithet();
2499         }
2500
2501         /*if (epithetName != null && !synonymsEpithet.contains(epithetName)){
2502             synonymsEpithet.add(epithetName);
2503         }*/
2504
2505         //create potential combinations...
2506         inferredSynName = TaxonNameFactory.NewZoologicalInstance(syn.getName().getRank());
2507
2508         inferredSynName.setGenusOrUninomial(synParentGenus);
2509         if (zooSynName.isSpecies()){
2510               inferredSynName.setSpecificEpithet(synTaxonSpecificEpithet);
2511               if (parentSynZooName.isInfraGeneric()){
2512                   inferredSynName.setInfraGenericEpithet(synParentInfragenericName);
2513               }
2514         }
2515         if (zooSynName.isInfraSpecific()){
2516             inferredSynName.setSpecificEpithet(synParentSpecificEpithet);
2517             inferredSynName.setInfraSpecificEpithet(synTaxonInfraSpecificName);
2518         }
2519         if (parentSynZooName.isInfraGeneric()){
2520             inferredSynName.setInfraGenericEpithet(synParentInfragenericName);
2521         }
2522
2523
2524         potentialCombination = Synonym.NewInstance(inferredSynName, null);
2525
2526         // Set the sourceReference
2527         potentialCombination.setSec(sourceReference);
2528
2529
2530         // Determine the idInSource
2531         String idInSourceSyn= getIdInSource(syn);
2532
2533         if (idInSourceParent != null && idInSourceSyn != null) {
2534             IdentifiableSource originalSource = IdentifiableSource.NewInstance(OriginalSourceType.Transformation, idInSourceSyn + "; " + idInSourceParent, POTENTIAL_COMBINATION_NAMESPACE, sourceReference, null);
2535             inferredSynName.addSource(originalSource);
2536             originalSource = IdentifiableSource.NewInstance(OriginalSourceType.Transformation, idInSourceSyn + "; " + idInSourceParent, POTENTIAL_COMBINATION_NAMESPACE, sourceReference, null);
2537             potentialCombination.addSource(originalSource);
2538         }
2539
2540         return potentialCombination;
2541     }
2542
2543     private Synonym createInferredGenus(Taxon taxon,
2544             Map<UUID, IZoologicalName> zooHashMap, IZoologicalName taxonName,
2545             String epithetOfTaxon, String genusOfTaxon,
2546             List<String> taxonNames, IZoologicalName zooParentName,
2547             TaxonBase syn) {
2548
2549         Synonym inferredGenus;
2550         TaxonName synName;
2551         IZoologicalName inferredSynName;
2552         synName =syn.getName();
2553         HibernateProxyHelper.deproxy(syn);
2554
2555         // Determine the idInSource
2556         String idInSourceSyn = getIdInSource(syn);
2557         String idInSourceTaxon = getIdInSource(taxon);
2558         // Determine the sourceReference
2559         Reference sourceReference = syn.getSec();
2560
2561         //logger.warn(sourceReference.getTitleCache());
2562
2563         synName = syn.getName();
2564         IZoologicalName synZooName = getZoologicalName(synName.getUuid(), zooHashMap);
2565         String synSpeciesEpithetName = synZooName.getSpecificEpithet();
2566                      /* if (synonymsEpithet != null && !synonymsEpithet.contains(synSpeciesEpithetName)){
2567             synonymsEpithet.add(synSpeciesEpithetName);
2568         }*/
2569
2570         inferredSynName = TaxonNameFactory.NewZoologicalInstance(taxon.getName().getRank());
2571         //TODO:differ between parent is genus and taxon is species, parent is subgenus and taxon is species, parent is species and taxon is subspecies and parent is genus and taxon is subgenus...
2572
2573
2574         inferredSynName.setGenusOrUninomial(genusOfTaxon);
2575         if (zooParentName.isInfraGeneric()){
2576             inferredSynName.setInfraGenericEpithet(zooParentName.getInfraGenericEpithet());
2577         }
2578
2579         if (taxonName.isSpecies()){
2580             inferredSynName.setSpecificEpithet(synSpeciesEpithetName);
2581         }
2582         if (taxonName.isInfraSpecific()){
2583             inferredSynName.setSpecificEpithet(epithetOfTaxon);
2584             inferredSynName.setInfraSpecificEpithet(synZooName.getInfraGenericEpithet());
2585         }
2586
2587
2588         inferredGenus = Synonym.NewInstance(inferredSynName, null);
2589
2590         // Set the sourceReference
2591         inferredGenus.setSec(sourceReference);
2592
2593         // Add the original source
2594         if (idInSourceSyn != null && idInSourceTaxon != null) {
2595             IdentifiableSource originalSource = IdentifiableSource.NewInstance(OriginalSourceType.Transformation,
2596                     idInSourceSyn + "; " + idInSourceTaxon, INFERRED_GENUS_NAMESPACE, sourceReference, null);
2597             inferredGenus.addSource(originalSource);
2598
2599             originalSource = IdentifiableSource.NewInstance(OriginalSourceType.Transformation,
2600                     idInSourceSyn + "; " + idInSourceTaxon, INFERRED_GENUS_NAMESPACE, sourceReference, null);
2601             inferredSynName.addSource(originalSource);
2602             originalSource = null;
2603
2604         }else{
2605             logger.error("There is an idInSource missing: " + idInSourceSyn + " of Synonym or " + idInSourceTaxon + " of Taxon");
2606             IdentifiableSource originalSource = IdentifiableSource.NewInstance(OriginalSourceType.Transformation,
2607                     idInSourceSyn + "; " + idInSourceTaxon, INFERRED_GENUS_NAMESPACE, sourceReference, null);
2608             inferredGenus.addSource(originalSource);
2609
2610             originalSource = IdentifiableSource.NewInstance(OriginalSourceType.Transformation,
2611                     idInSourceSyn + "; " + idInSourceTaxon, INFERRED_GENUS_NAMESPACE, sourceReference, null);
2612             inferredSynName.addSource(originalSource);
2613             originalSource = null;
2614         }
2615
2616         taxon.addSynonym(inferredGenus, SynonymType.INFERRED_GENUS_OF());
2617
2618         return inferredGenus;
2619     }
2620
2621     private Synonym createInferredEpithets(Taxon taxon,
2622             Map<UUID, IZoologicalName> zooHashMap, IZoologicalName taxonName,
2623             String epithetOfTaxon, String infragenericEpithetOfTaxon,
2624             String infraspecificEpithetOfTaxon, List<String> taxonNames,
2625             TaxonName parentName, TaxonBase<?> syn) {
2626
2627         Synonym inferredEpithet;
2628         TaxonName synName;
2629         IZoologicalName inferredSynName;
2630         HibernateProxyHelper.deproxy(syn);
2631
2632         // Determine the idInSource
2633         String idInSourceSyn = getIdInSource(syn);
2634         String idInSourceTaxon =  getIdInSource(taxon);
2635         // Determine the sourceReference
2636         Reference sourceReference = syn.getSec();
2637
2638         if (sourceReference == null){
2639              logger.warn("The synonym has no sec reference because it is a misapplied name! Take the sec reference of taxon" + taxon.getSec());
2640              sourceReference = taxon.getSec();
2641         }
2642
2643         synName = syn.getName();
2644         IZoologicalName zooSynName = getZoologicalName(synName.getUuid(), zooHashMap);
2645         String synGenusName = zooSynName.getGenusOrUninomial();
2646         String synInfraGenericEpithet = null;
2647         String synSpecificEpithet = null;
2648
2649         if (zooSynName.getInfraGenericEpithet() != null){
2650             synInfraGenericEpithet = zooSynName.getInfraGenericEpithet();
2651         }
2652
2653         if (zooSynName.isInfraSpecific()){
2654             synSpecificEpithet = zooSynName.getSpecificEpithet();
2655         }
2656
2657                      /* if (synGenusName != null && !synonymsGenus.containsKey(synGenusName)){
2658             synonymsGenus.put(synGenusName, idInSource);
2659         }*/
2660
2661         inferredSynName = TaxonNameFactory.NewZoologicalInstance(taxon.getName().getRank());
2662
2663         // DEBUG TODO: for subgenus or subspecies the infrageneric or infraspecific epithet should be used!!!
2664         if (epithetOfTaxon == null && infragenericEpithetOfTaxon == null && infraspecificEpithetOfTaxon == null) {
2665             logger.error("This specificEpithet is NULL" + taxon.getTitleCache());
2666         }
2667         inferredSynName.setGenusOrUninomial(synGenusName);
2668
2669         if (parentName.isInfraGeneric()){
2670             inferredSynName.setInfraGenericEpithet(synInfraGenericEpithet);
2671         }
2672         if (taxonName.isSpecies()){
2673             inferredSynName.setSpecificEpithet(epithetOfTaxon);
2674         }else if (taxonName.isInfraSpecific()){
2675             inferredSynName.setSpecificEpithet(synSpecificEpithet);
2676             inferredSynName.setInfraSpecificEpithet(infraspecificEpithetOfTaxon);
2677         }
2678
2679         inferredEpithet = Synonym.NewInstance(inferredSynName, null);
2680
2681         // Set the sourceReference
2682         inferredEpithet.setSec(sourceReference);
2683
2684         /* Add the original source
2685         if (idInSource != null) {
2686             IdentifiableSource originalSource = IdentifiableSource.NewInstance(idInSource, "InferredEpithetOf", syn.getSec(), null);
2687
2688             // Add the citation
2689             Reference citation = getCitation(syn);
2690             if (citation != null) {
2691                 originalSource.setCitation(citation);
2692                 inferredEpithet.addSource(originalSource);
2693             }
2694         }*/
2695         String taxonId = idInSourceTaxon+ "; " + idInSourceSyn;
2696
2697
2698         IdentifiableSource originalSource = IdentifiableSource.NewInstance(OriginalSourceType.Transformation,
2699                 taxonId, INFERRED_EPITHET_NAMESPACE, sourceReference, null);
2700
2701         inferredEpithet.addSource(originalSource);
2702
2703         originalSource = IdentifiableSource.NewInstance(OriginalSourceType.Transformation,
2704                 taxonId, INFERRED_EPITHET_NAMESPACE, sourceReference, null);
2705
2706         inferredSynName.addSource(originalSource);
2707
2708
2709
2710         taxon.addSynonym(inferredEpithet, SynonymType.INFERRED_EPITHET_OF());
2711
2712         return inferredEpithet;
2713     }
2714
2715     /**
2716      * Returns an existing IZoologicalName or extends an internal hashmap if it does not exist.
2717      * Very likely only useful for createInferredSynonyms().
2718      * @param uuid
2719      * @param zooHashMap
2720      * @return
2721      */
2722     private IZoologicalName getZoologicalName(UUID uuid, Map <UUID, IZoologicalName> zooHashMap) {
2723         IZoologicalName taxonName =nameDao.findZoologicalNameByUUID(uuid);
2724         if (taxonName == null) {
2725             taxonName = zooHashMap.get(uuid);
2726         }
2727         return taxonName;
2728     }
2729
2730     /**
2731      * Returns the idInSource for a given Synonym.
2732      * @param syn
2733      */
2734     private String getIdInSource(TaxonBase<?> taxonBase) {
2735         String idInSource = null;
2736         Set<IdentifiableSource> sources = taxonBase.getSources();
2737         if (sources.size() == 1) {
2738             IdentifiableSource source = sources.iterator().next();
2739             if (source != null) {
2740                 idInSource  = source.getIdInSource();
2741             }
2742         } else if (sources.size() > 1) {
2743             int count = 1;
2744             idInSource = "";
2745             for (IdentifiableSource source : sources) {
2746                 idInSource += source.getIdInSource();
2747                 if (count < sources.size()) {
2748                     idInSource += "; ";
2749                 }
2750                 count++;
2751             }
2752         } else if (sources.size() == 0){
2753             logger.warn("No idInSource for TaxonBase " + taxonBase.getUuid() + " - " + taxonBase.getTitleCache());
2754         }
2755
2756
2757         return idInSource;
2758     }
2759
2760
2761     /**
2762      * Returns the citation for a given Synonym.
2763      * @param syn
2764      */
2765     private Reference getCitation(Synonym syn) {
2766         Reference citation = null;
2767         Set<IdentifiableSource> sources = syn.getSources();
2768         if (sources.size() == 1) {
2769             IdentifiableSource source = sources.iterator().next();
2770             if (source != null) {
2771                 citation = source.getCitation();
2772             }
2773         } else if (sources.size() > 1) {
2774             logger.warn("This Synonym has more than one source: " + syn.getUuid() + " (" + syn.getTitleCache() +")");
2775         }
2776
2777         return citation;
2778     }
2779
2780     @Override
2781     public List<Synonym>  createAllInferredSynonyms(Taxon taxon, Classification tree, boolean doWithMisappliedNames){
2782         List <Synonym> inferredSynonyms = new ArrayList<>();
2783
2784         inferredSynonyms.addAll(createInferredSynonyms(taxon, tree, SynonymType.INFERRED_EPITHET_OF(), doWithMisappliedNames));
2785         inferredSynonyms.addAll(createInferredSynonyms(taxon, tree, SynonymType.INFERRED_GENUS_OF(), doWithMisappliedNames));
2786         inferredSynonyms.addAll(createInferredSynonyms(taxon, tree, SynonymType.POTENTIAL_COMBINATION_OF(), doWithMisappliedNames));
2787
2788         return inferredSynonyms;
2789     }
2790
2791     @Override
2792     public List<Classification> listClassifications(TaxonBase taxonBase, Integer limit, Integer start, List<String> propertyPaths) {
2793
2794         // TODO quickly implemented, create according dao !!!!
2795         Set<TaxonNode> nodes = new HashSet<>();
2796         Set<Classification> classifications = new HashSet<>();
2797         List<Classification> list = new ArrayList<>();
2798
2799         if (taxonBase == null) {
2800             return list;
2801         }
2802
2803         taxonBase = load(taxonBase.getUuid());
2804
2805         if (taxonBase instanceof Taxon) {
2806             nodes.addAll(((Taxon)taxonBase).getTaxonNodes());
2807         } else {
2808             Taxon taxon = ((Synonym)taxonBase).getAcceptedTaxon();
2809             if (taxon != null){
2810                 nodes.addAll(taxon.getTaxonNodes());
2811             }
2812         }
2813         for (TaxonNode node : nodes) {
2814             classifications.add(node.getClassification());
2815         }
2816         list.addAll(classifications);
2817         return list;
2818     }
2819
2820     @Override
2821     @Transactional(readOnly = false)
2822     public UpdateResult changeRelatedTaxonToSynonym(UUID fromTaxonUuid,
2823             UUID toTaxonUuid,
2824             TaxonRelationshipType oldRelationshipType,
2825             SynonymType synonymType) throws DataChangeNoRollbackException {
2826         UpdateResult result = new UpdateResult();
2827         Taxon fromTaxon = (Taxon) dao.load(fromTaxonUuid);
2828         Taxon toTaxon = (Taxon) dao.load(toTaxonUuid);
2829         result = changeRelatedTaxonToSynonym(fromTaxon, toTaxon, oldRelationshipType, synonymType);
2830
2831         result.addUpdatedObject(fromTaxon);
2832         result.addUpdatedObject(toTaxon);
2833         result.addUpdatedObject(result.getCdmEntity());
2834
2835         return result;
2836     }
2837
2838     @Override
2839     @Transactional(readOnly = false)
2840     public UpdateResult changeRelatedTaxonToSynonym(Taxon fromTaxon, Taxon toTaxon, TaxonRelationshipType oldRelationshipType,
2841             SynonymType synonymType) throws DataChangeNoRollbackException {
2842
2843         UpdateResult result = new UpdateResult();
2844         // Create new synonym using concept name
2845         TaxonName synonymName = fromTaxon.getName();
2846
2847         // Remove concept relation from taxon
2848         toTaxon.removeTaxon(fromTaxon, oldRelationshipType);
2849
2850         // Create a new synonym for the taxon
2851         Synonym synonym;
2852         if (synonymType != null
2853                 && synonymType.equals(SynonymType.HOMOTYPIC_SYNONYM_OF())){
2854             synonym = Synonym.NewInstance(synonymName, fromTaxon.getSec());
2855             toTaxon.addHomotypicSynonym(synonym);
2856         } else{
2857             synonym = toTaxon.addHeterotypicSynonymName(synonymName);
2858         }
2859
2860         this.saveOrUpdate(toTaxon);
2861         //TODO: configurator and classification
2862         TaxonDeletionConfigurator config = new TaxonDeletionConfigurator();
2863         config.setDeleteNameIfPossible(false);
2864         result.includeResult(this.deleteTaxon(fromTaxon.getUuid(), config, null));
2865         result.setCdmEntity(synonym);
2866         result.addUpdatedObject(toTaxon);
2867         result.addUpdatedObject(synonym);
2868         return result;
2869     }
2870
2871     @Override
2872     public DeleteResult isDeletable(UUID taxonBaseUuid, DeleteConfiguratorBase config){
2873         DeleteResult result = new DeleteResult();
2874         TaxonBase<?> taxonBase = load(taxonBaseUuid);
2875         Set<CdmBase> references = commonService.getReferencingObjectsForDeletion(taxonBase);
2876         if (taxonBase instanceof Taxon){
2877             TaxonDeletionConfigurator taxonConfig = (TaxonDeletionConfigurator) config;
2878             result = isDeletableForTaxon(references, taxonConfig);
2879         }else{
2880             SynonymDeletionConfigurator synonymConfig = (SynonymDeletionConfigurator) config;
2881             result = isDeletableForSynonym(references, synonymConfig);
2882         }
2883         return result;
2884     }
2885
2886     private DeleteResult isDeletableForSynonym(Set<CdmBase> references, SynonymDeletionConfigurator config){
2887         String message;
2888         DeleteResult result = new DeleteResult();
2889         for (CdmBase ref: references){
2890             if (!(ref instanceof Taxon || ref instanceof TaxonName )){
2891                 message = "The Synonym can't be deleted as long as it is referenced by " + ref.getClass().getSimpleName() + " with id "+ ref.getId();
2892                 result.addException(new ReferencedObjectUndeletableException(message));
2893                 result.addRelatedObject(ref);
2894                 result.setAbort();
2895             }
2896         }
2897
2898         return result;
2899     }
2900
2901     private DeleteResult isDeletableForTaxon(Set<CdmBase> references, TaxonDeletionConfigurator config){
2902         String message = null;
2903         DeleteResult result = new DeleteResult();
2904         for (CdmBase ref: references){
2905             if (!(ref instanceof TaxonName)){
2906                 message = null;
2907                 if (!config.isDeleteSynonymRelations() && (ref instanceof Synonym)){
2908                     message = "The taxon can't be deleted as long as it has synonyms.";
2909                 }
2910                 if (!config.isDeleteDescriptions() && (ref instanceof DescriptionBase)){
2911                     message = "The taxon can't be deleted as long as it has factual data.";
2912                 }
2913
2914                 if (!config.isDeleteTaxonNodes() && (ref instanceof TaxonNode)){
2915                     message = "The taxon can't be deleted as long as it belongs to a taxon node.";
2916                 }
2917                 if (!config.isDeleteTaxonRelationships() && (ref instanceof TaxonRelationship)){
2918                     if (!config.isDeleteMisappliedNamesAndInvalidDesignations() &&
2919                             (((TaxonRelationship)ref).getType().isMisappliedNameOrInvalidDesignation())){
2920                         message = "The taxon can't be deleted as long as it has misapplied names or invalid designations.";
2921                     } else{
2922                         message = "The taxon can't be deleted as long as it belongs to taxon relationship.";
2923                     }
2924                 }
2925                 if (ref instanceof PolytomousKeyNode){
2926                     message = "The taxon can't be deleted as long as it is referenced by a polytomous key node.";
2927                 }
2928
2929                 if (HibernateProxyHelper.isInstanceOf(ref, IIdentificationKey.class)){
2930                    message = "Taxon can't be deleted as it is used in an identification key. Remove from identification key prior to deleting this taxon";
2931                 }
2932
2933
2934                /* //PolytomousKeyNode
2935                 if (referencingObject.isInstanceOf(PolytomousKeyNode.class)){
2936                     String message = "Taxon" + taxon.getTitleCache() + " can't be deleted as it is used in polytomous key node";
2937                     return message;
2938                 }*/
2939
2940                 //TaxonInteraction
2941                 if (ref.isInstanceOf(TaxonInteraction.class)){
2942                     message = "Taxon can't be deleted as it is used in taxonInteraction#taxon2";
2943                 }
2944
2945               //TaxonInteraction
2946                 if (ref.isInstanceOf(DeterminationEvent.class)){
2947                     message = "Taxon can't be deleted as it is used in a determination event";
2948                 }
2949             }
2950             if (message != null){
2951                     result.addException(new ReferencedObjectUndeletableException(message));
2952                     result.addRelatedObject(ref);
2953                     result.setAbort();
2954             }
2955         }
2956
2957         return result;
2958     }
2959
2960     @Override
2961     public IncludedTaxaDTO listIncludedTaxa(UUID taxonUuid, IncludedTaxonConfiguration config) {
2962         IncludedTaxaDTO result = new IncludedTaxaDTO(taxonUuid);
2963
2964         //preliminary implementation
2965
2966         Set<Taxon> taxa = new HashSet<>();
2967         TaxonBase<?> taxonBase = find(taxonUuid);
2968         if (taxonBase == null){
2969             return new IncludedTaxaDTO();
2970         }else if (taxonBase.isInstanceOf(Taxon.class)){
2971             Taxon taxon = CdmBase.deproxy(taxonBase, Taxon.class);
2972             taxa.add(taxon);
2973         }else if (taxonBase.isInstanceOf(Synonym.class)){
2974             //TODO partial synonyms ??
2975             //TODO synonyms in general
2976             Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
2977             taxa.add(syn.getAcceptedTaxon());
2978         }else{
2979             throw new IllegalArgumentException("Unhandled class " + taxonBase.getClass().getSimpleName());
2980         }
2981
2982         Set<Taxon> related = makeRelatedIncluded(taxa, result, config);
2983         int i = 0;
2984         while((! related.isEmpty()) && i++ < 100){  //to avoid
2985              related = makeRelatedIncluded(related, result, config);
2986         }
2987
2988         return result;
2989     }
2990
2991     /**
2992      * Computes all children and conceptually congruent and included taxa and adds them to the existingTaxa
2993      * data structure.
2994      * @return the set of conceptually related taxa for further use
2995      */
2996     /**
2997      * @param uncheckedTaxa
2998      * @param existingTaxa
2999      * @param config
3000      * @return
3001      */
3002     private Set<Taxon> makeRelatedIncluded(Set<Taxon> uncheckedTaxa, IncludedTaxaDTO existingTaxa, IncludedTaxonConfiguration config) {
3003
3004         //children
3005         Set<TaxonNode> taxonNodes = new HashSet<>();
3006         for (Taxon taxon: uncheckedTaxa){
3007             taxonNodes.addAll(taxon.getTaxonNodes());
3008         }
3009
3010         Set<Taxon> children = new HashSet<>();
3011         if (! config.onlyCongruent){
3012             for (TaxonNode node: taxonNodes){
3013                 List<TaxonNode> childNodes = nodeService.loadChildNodesOfTaxonNode(node, null, true, config.includeUnpublished, null);
3014                 for (TaxonNode child : childNodes){
3015                     children.add(child.getTaxon());
3016                 }
3017             }
3018             children.remove(null);  // just to be on the save side
3019         }
3020
3021         Iterator<Taxon> it = children.iterator();
3022         while(it.hasNext()){
3023             UUID uuid = it.next().getUuid();
3024             if (existingTaxa.contains(uuid)){
3025                 it.remove();
3026             }else{
3027                 existingTaxa.addIncludedTaxon(uuid, new ArrayList<>(), false);
3028             }
3029         }
3030
3031         //concept relations
3032         Set<Taxon> uncheckedAndChildren = new HashSet<>(uncheckedTaxa);
3033         uncheckedAndChildren.addAll(children);
3034
3035         Set<Taxon> relatedTaxa = makeConceptIncludedTaxa(uncheckedAndChildren, existingTaxa, config);
3036
3037
3038         Set<Taxon> result = new HashSet<>(relatedTaxa);
3039         return result;
3040     }
3041
3042     /**
3043      * Computes all conceptually congruent or included taxa and adds them to the existingTaxa data structure.
3044      * @return the set of these computed taxa
3045      */
3046     private Set<Taxon> makeConceptIncludedTaxa(Set<Taxon> unchecked, IncludedTaxaDTO existingTaxa, IncludedTaxonConfiguration config) {
3047         Set<Taxon> result = new HashSet<>();
3048
3049         for (Taxon taxon : unchecked){
3050             Set<TaxonRelationship> fromRelations = taxon.getRelationsFromThisTaxon();
3051             Set<TaxonRelationship> toRelations = taxon.getRelationsToThisTaxon();
3052
3053             for (TaxonRelationship fromRel : fromRelations){
3054                 if (config.includeDoubtful == false && fromRel.isDoubtful()){
3055                     continue;
3056                 }
3057                 if (fromRel.getType().equals(TaxonRelationshipType.CONGRUENT_TO()) ||
3058                         !config.onlyCongruent && fromRel.getType().equals(TaxonRelationshipType.INCLUDES()) ||
3059                         !config.onlyCongruent && fromRel.getType().equals(TaxonRelationshipType.CONGRUENT_OR_INCLUDES())
3060                         ){
3061                     result.add(fromRel.getToTaxon());
3062                 }
3063             }
3064
3065             for (TaxonRelationship toRel : toRelations){
3066                 if (config.includeDoubtful == false && toRel.isDoubtful()){
3067                     continue;
3068                 }
3069                 if (toRel.getType().equals(TaxonRelationshipType.CONGRUENT_TO())){
3070                     result.add(toRel.getFromTaxon());
3071                 }
3072             }
3073         }
3074
3075         Iterator<Taxon> it = result.iterator();
3076         while(it.hasNext()){
3077             UUID uuid = it.next().getUuid();
3078             if (existingTaxa.contains(uuid)){
3079                 it.remove();
3080             }else{
3081                 existingTaxa.addIncludedTaxon(uuid, new ArrayList<>(), false);
3082             }
3083         }
3084         return result;
3085     }
3086
3087     @Override
3088     public List<TaxonBase> findTaxaByName(MatchingTaxonConfigurator config){
3089         List<TaxonBase> taxonList = dao.getTaxaByName(true, config.isIncludeSynonyms(), false, false, false,
3090                 config.getTaxonNameTitle(), null, null, MatchMode.EXACT, null, config.isIncludeSynonyms(), null, 0, 0, config.getPropertyPath());
3091         return taxonList;
3092     }
3093
3094         @Override
3095         @Transactional(readOnly = true)
3096         public <S extends TaxonBase> Pager<IdentifiedEntityDTO<S>> findByIdentifier(
3097                         Class<S> clazz, String identifier, DefinedTerm identifierType, TaxonNode subtreeFilter,
3098                         MatchMode matchmode, boolean includeEntity, Integer pageSize,
3099                         Integer pageNumber,     List<String> propertyPaths) {
3100                 if (subtreeFilter == null){
3101                         return findByIdentifier(clazz, identifier, identifierType, matchmode, includeEntity, pageSize, pageNumber, propertyPaths);
3102                 }
3103
3104                 long numberOfResults = dao.countByIdentifier(clazz, identifier, identifierType, subtreeFilter, matchmode);
3105         List<Object[]> daoResults = new ArrayList<>();
3106         if(numberOfResults > 0) { // no point checking again
3107                 daoResults = dao.findByIdentifier(clazz, identifier, identifierType, subtreeFilter,
3108                                 matchmode, includeEntity, pageSize, pageNumber, propertyPaths);
3109         }
3110
3111         List<IdentifiedEntityDTO<S>> result = new ArrayList<>();
3112         for (Object[] daoObj : daoResults){
3113                 if (includeEntity){
3114                         result.add(new IdentifiedEntityDTO<>((DefinedTerm)daoObj[0], (String)daoObj[1], (S)daoObj[2]));
3115                 }else{
3116                         result.add(new IdentifiedEntityDTO<>((DefinedTerm)daoObj[0], (String)daoObj[1], (UUID)daoObj[2], (String)daoObj[3], null));
3117                 }
3118         }
3119                 return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, result);
3120         }
3121
3122         @Override
3123     @Transactional(readOnly = true)
3124     public <S extends TaxonBase> Pager<MarkedEntityDTO<S>> findByMarker(
3125             Class<S> clazz, MarkerType markerType, Boolean markerValue,
3126             TaxonNode subtreeFilter, boolean includeEntity, TaxonTitleType titleType,
3127             Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
3128         if (subtreeFilter == null){
3129             return super.findByMarker (clazz, markerType, markerValue, includeEntity, pageSize, pageNumber, propertyPaths);
3130         }
3131
3132         Long numberOfResults = dao.countByMarker(clazz, markerType, markerValue, subtreeFilter);
3133         List<Object[]> daoResults = new ArrayList<>();
3134         if(numberOfResults > 0) { // no point checking again
3135             daoResults = dao.findByMarker(clazz, markerType, markerValue, subtreeFilter,
3136                     includeEntity, titleType, pageSize, pageNumber, propertyPaths);
3137         }
3138
3139         List<MarkedEntityDTO<S>> result = new ArrayList<>();
3140         for (Object[] daoObj : daoResults){
3141             if (includeEntity){
3142                 result.add(new MarkedEntityDTO<S>((MarkerType)daoObj[0], (Boolean)daoObj[1], (S)daoObj[2]));
3143             }else{
3144                 result.add(new MarkedEntityDTO<S>((MarkerType)daoObj[0], (Boolean)daoObj[1], (UUID)daoObj[2], (String)daoObj[3]));
3145             }
3146         }
3147         return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, result);
3148     }
3149
3150     @Override
3151         @Transactional(readOnly = false)
3152         public UpdateResult moveSynonymToAnotherTaxon(Synonym oldSynonym, UUID newTaxonUUID, boolean moveHomotypicGroup,
3153             SynonymType newSynonymType, Reference newSecundum, String newSecundumDetail,
3154             boolean keepSecundumIfUndefined) throws HomotypicalGroupChangeException {
3155
3156             UpdateResult result = new UpdateResult();
3157                 Taxon newTaxon = CdmBase.deproxy(dao.load(newTaxonUUID),Taxon.class);
3158                 result = moveSynonymToAnotherTaxon(oldSynonym, newTaxon, moveHomotypicGroup, newSynonymType,
3159                         newSecundum, newSecundumDetail, keepSecundumIfUndefined);
3160
3161                 return result;
3162         }
3163
3164         @Override
3165         public UpdateResult moveFactualDateToAnotherTaxon(UUID fromTaxonUuid, UUID toTaxonUuid){
3166                 UpdateResult result = new UpdateResult();
3167
3168                 Taxon fromTaxon = (Taxon)dao.load(fromTaxonUuid);
3169                 Taxon toTaxon = (Taxon) dao.load(toTaxonUuid);
3170         for(TaxonDescription description : fromTaxon.getDescriptions()){
3171               //reload to avoid session conflicts
3172               description = HibernateProxyHelper.deproxy(description, TaxonDescription.class);
3173
3174               String moveMessage = String.format("Description moved from %s", fromTaxon);
3175               if(description.isProtectedTitleCache()){
3176                   String separator = "";
3177                   if(!StringUtils.isBlank(description.getTitleCache())){
3178                       separator = " - ";
3179                   }
3180                   description.setTitleCache(description.getTitleCache() + separator + moveMessage, true);
3181               }
3182               Annotation annotation = Annotation.NewInstance(moveMessage, Language.getDefaultLanguage());
3183               annotation.setAnnotationType(AnnotationType.TECHNICAL());
3184               description.addAnnotation(annotation);
3185               toTaxon.addDescription(description);
3186               dao.saveOrUpdate(toTaxon);
3187               dao.saveOrUpdate(fromTaxon);
3188               result.addUpdatedObject(toTaxon);
3189               result.addUpdatedObject(fromTaxon);
3190
3191         }
3192
3193         return result;
3194         }
3195
3196         @Override
3197         @Transactional(readOnly = false)
3198         public UpdateResult swapSynonymAndAcceptedTaxon(UUID synonymUUid,
3199                         UUID acceptedTaxonUuid) {
3200                 TaxonBase<?> base = this.load(synonymUUid);
3201                 Synonym syn = HibernateProxyHelper.deproxy(base, Synonym.class);
3202                 base = this.load(acceptedTaxonUuid);
3203                 Taxon taxon = HibernateProxyHelper.deproxy(base, Taxon.class);
3204
3205                 return this.swapSynonymAndAcceptedTaxon(syn, taxon);
3206         }
3207
3208     /**
3209      * {@inheritDoc}
3210      */
3211     @Override
3212     public TaxonRelationshipsDTO listTaxonRelationships(UUID taxonUuid, Set<TaxonRelationshipType> directTypes,
3213             Set<TaxonRelationshipType> inversTypes,
3214             Direction direction, boolean groupMisapplications,
3215             boolean includeUnpublished,
3216             Integer pageSize, Integer pageNumber) {
3217         TaxonBase<?> taxonBase = dao.load(taxonUuid);
3218         if (taxonBase == null || !taxonBase.isInstanceOf(TaxonBase.class)){
3219             //TODO handle
3220             throw new RuntimeException("Taxon for uuid " + taxonUuid + " not found");
3221         }else{
3222             Taxon taxon = CdmBase.deproxy(taxonBase, Taxon.class);
3223             boolean doDirect = (direction == null || direction == Direction.relatedTo);
3224             boolean doInvers = (direction == null || direction == Direction.relatedFrom);
3225
3226             TaxonRelationshipsDTO dto = new TaxonRelationshipsDTO();
3227
3228             //TODO paging is difficult because misapplication string is an attribute
3229             //of toplevel dto
3230 //        long numberOfResults = dao.countTaxonRelationships(taxon, type, includeUnpublished, TaxonRelationship.Direction.relatedFrom);
3231 //        List<TaxonRelationshipsDTO> results = new ArrayList<>();
3232 //        if(numberOfResults > 0) { // no point checking again
3233 //            results = dao.getTaxonRelationships(taxon, type, includeUnpublished, pageSize, pageNumber, orderHints, propertyPaths, TaxonRelationship.Direction.relatedFrom);
3234 //        }
3235 //
3236 //        return new DefaultPagerImpl<>(pageNumber, numberOfResults, pageSize, results);;
3237
3238             //TODO languages
3239             List<Language> languages = null;
3240             if (doDirect){
3241                 direction = Direction.relatedTo;
3242                 //TODO order hints, property path
3243                 List<TaxonRelationship> relations = dao.getTaxonRelationships(taxon, directTypes, includeUnpublished, pageSize, pageNumber, null, null, direction.invers());
3244                 for (TaxonRelationship relation : relations){
3245                     dto.addRelation(relation, direction, languages);
3246                 }
3247             }
3248             if (doInvers){
3249                 direction = Direction.relatedFrom;
3250                 //TODO order hints, property path
3251                 List<TaxonRelationship> relations = dao.getTaxonRelationships(taxon, inversTypes, includeUnpublished, pageSize, pageNumber, null, null, direction.invers());
3252                 for (TaxonRelationship relation : relations){
3253                     dto.addRelation(relation, direction, languages);
3254                 }
3255             }
3256             if (groupMisapplications){
3257                 //TODO
3258                 dto.createMisapplicationString();
3259             }
3260             return dto;
3261         }
3262     }
3263
3264 }