cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/TaxonServiceSearchTest.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import static org.junit.Assert.assertNotNull;
  13
  14 import java.io.FileNotFoundException;
  15 import java.io.IOException;
  16 import java.util.ArrayList;
  17 import java.util.Arrays;
  18 import java.util.EnumSet;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.RandomStringUtils;
  26 import org.apache.log4j.Level;
  27 import org.apache.log4j.Logger;
  28 import org.apache.lucene.document.Document;
  29 import org.junit.Assert;
  30 import org.junit.Before;
  31 import org.junit.Ignore;
  32 import org.junit.Test;
  33 import org.unitils.dbunit.annotation.DataSet;
  34 import org.unitils.spring.annotation.SpringBeanByType;
  35
  36 import eu.etaxonomy.cdm.api.service.config.FindTaxaAndNamesConfiguratorImpl;
  37 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  38 import eu.etaxonomy.cdm.api.service.pager.Pager;
  39 import eu.etaxonomy.cdm.api.service.search.ICdmMassIndexer;
  40 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  41 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  42 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  43 import eu.etaxonomy.cdm.common.UTF8;
  44 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  45 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  46 import eu.etaxonomy.cdm.model.common.CdmBase;
  47 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  48 import eu.etaxonomy.cdm.model.common.Language;
  49 import eu.etaxonomy.cdm.model.description.CategoricalData;
  50 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  51 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  52 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  53 import eu.etaxonomy.cdm.model.description.Distribution;
  54 import eu.etaxonomy.cdm.model.description.Feature;
  55 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  56 import eu.etaxonomy.cdm.model.description.State;
  57 import eu.etaxonomy.cdm.model.description.StateData;
  58 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  59 import eu.etaxonomy.cdm.model.description.TextData;
  60 import eu.etaxonomy.cdm.model.location.Country;
  61 import eu.etaxonomy.cdm.model.location.NamedArea;
  62 import eu.etaxonomy.cdm.model.name.IBotanicalName;
  63 import eu.etaxonomy.cdm.model.name.Rank;
  64 import eu.etaxonomy.cdm.model.name.TaxonName;
  65 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  66 import eu.etaxonomy.cdm.model.reference.Reference;
  67 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  68 import eu.etaxonomy.cdm.model.taxon.Classification;
  69 import eu.etaxonomy.cdm.model.taxon.Synonym;
  70 import eu.etaxonomy.cdm.model.taxon.SynonymType;
  71 import eu.etaxonomy.cdm.model.taxon.Taxon;
  72 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  73 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  74 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  75 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  76 import eu.etaxonomy.cdm.model.term.DefinedTermBase;
  77 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  78 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  79 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  80 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  81
  82 /**
  83  * @author a.babadshanjan, a.kohlbecker
  84  * @since 04.02.2009
  85  */
  86 public class TaxonServiceSearchTest extends CdmTransactionalIntegrationTest {
  87
  88     private static Logger logger = Logger.getLogger(TaxonServiceSearchTest.class);
  89
  90     private static final UUID ABIES_BALSAMEA_UUID = UUID.fromString("f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8");
  91     private static final UUID ABIES_ALBA_UUID = UUID.fromString("7dbd5810-a3e5-44b6-b563-25152b8867f4");
  92     private static final UUID CLASSIFICATION_UUID = UUID.fromString("2a5ceebb-4830-4524-b330-78461bf8cb6b");
  93     private static final UUID CLASSIFICATION_ALT_UUID = UUID.fromString("d7c741e3-ae9e-4a7d-a566-9e3a7a0b51ce");
  94     private static final UUID ABIES_SUBALPINA_UUID = UUID.fromString("9fee273c-c819-4f1f-913a-cd910465df51");
  95     private static final UUID ABIES_LASIOCARPA_UUID = UUID.fromString("9ce1fecf-c1ad-4127-be01-85d5d9f847ce");
  96
  97     private static final UUID ROOTNODE_CLASSIFICATION_5000 = UUID.fromString("a8266e45-091f-432f-87ae-c625e6aa9bbc");
  98
  99     private static final UUID DESC_ABIES_BALSAMEA_UUID = UUID.fromString("900108d8-e6ce-495e-b32e-7aad3099135e");
 100     private static final UUID DESC_ABIES_ALBA_UUID = UUID.fromString("ec8bba03-d993-4c85-8472-18b14942464b");
 101     private static final UUID DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID = UUID.fromString("e9d8c2fd-6409-46d5-9c2e-14a2bbb1b2b1");
 102
 103
 104     private static final int NUM_OF_NEW_RADOM_ENTITIES = 1000;
 105
 106     private boolean includeUnpublished = true;
 107
 108
 109
 110     @SpringBeanByType
 111     private ITaxonService taxonService;
 112     @SpringBeanByType
 113     private ITermService termService;
 114     @SpringBeanByType
 115     private IClassificationService classificationService;
 116     @SpringBeanByType
 117     private IReferenceService referenceService;
 118     @SpringBeanByType
 119     private IDescriptionService descriptionService;
 120     @SpringBeanByType
 121     private INameService nameService;
 122     @SpringBeanByType
 123     private ITaxonNodeService nodeService;
 124
 125     @SpringBeanByType
 126     private ICdmMassIndexer indexer;
 127
 128
 129     private static final int BENCHMARK_ROUNDS = 300;
 130
 131     private Set<Class<? extends CdmBase>> typesToIndex = null;
 132
 133     private NamedArea germany;
 134     private NamedArea france ;
 135     private NamedArea russia ;
 136     private NamedArea canada ;
 137
 138     @Before
 139     public void setUp() throws Exception {
 140         typesToIndex = new HashSet<>();
 141         typesToIndex.add(DescriptionElementBase.class);
 142         typesToIndex.add(TaxonBase.class);
 143         typesToIndex.add(TaxonRelationship.class);
 144
 145         germany =  Country.GERMANY();
 146         france = Country.FRANCEFRENCHREPUBLIC();
 147         russia = Country.RUSSIANFEDERATION();
 148         canada = Country.CANADA();
 149
 150         includeUnpublished = true;
 151     }
 152
 153     @Test
 154     public void testDbUnitUsageTest() throws Exception {
 155         assertNotNull("taxonService should exist", taxonService);
 156         assertNotNull("nameService should exist", nameService);
 157     }
 158
 159
 160     @SuppressWarnings("rawtypes")
 161     @Test
 162     @DataSet
 163     public final void testPurgeAndReindex() throws IOException, LuceneParseException {
 164
 165         refreshLuceneIndex();
 166         TaxonNode subtree = null;
 167
 168         Pager<SearchResult<TaxonBase>> pager = taxonService.findByFullText(null, "Abies", null, subtree,
 169                 includeUnpublished, null, true, null, null, null, null); // --> 8
 170         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 171
 172         indexer.purge(null);
 173         commitAndStartNewTransaction(null);
 174
 175         pager = taxonService.findByFullText(null, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 0
 176         Assert.assertEquals("Expecting no entities since the index has been purged", 0, pager.getCount().intValue());
 177
 178         indexer.reindex(indexer.indexedClasses(), null);
 179         commitAndStartNewTransaction(null);
 180
 181         pager = taxonService.findByFullText(null, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 8
 182         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 183     }
 184
 185
 186     @SuppressWarnings("rawtypes")
 187     @Test
 188     @DataSet
 189     public final void testFindByDescriptionElementFullText_CommonName() throws IOException,
 190             LuceneParseException {
 191
 192         TaxonNode subtree = null;
 193         refreshLuceneIndex();
 194
 195         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne",
 196                 null, subtree, null, null,
 197                 false, null, null, null, null);
 198         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1,
 199                 pager.getCount().intValue());
 200
 201         // the description containing the Nulltanne has no taxon attached,
 202         // taxon.id = null
 203         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Nulltanne", null, subtree, null, null,
 204                 false, null, null, null, null);
 205         Assert.assertEquals("Expecting no entity when searching for 'Nulltanne' ", 0, pager.getCount().intValue());
 206
 207         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, subtree, null,
 208                 Arrays.asList(new Language[] { Language.GERMAN() }), false, null, null, null, null);
 209         Assert.assertEquals("Expecting one entity when searching in German", 1, pager.getCount().intValue());
 210
 211         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, subtree, null,
 212                 Arrays.asList(new Language[] { Language.RUSSIAN() }), false, null, null, null, null);
 213         Assert.assertEquals("Expecting no entity when searching in Russian", 0, pager.getCount().intValue());
 214
 215     }
 216
 217     @SuppressWarnings("rawtypes")
 218     @Test
 219     @DataSet
 220     public final void testFindByDescriptionElementFullText_Distribution() throws IOException, LuceneParseException {
 221
 222         refreshLuceneIndex();
 223         TaxonNode subtree = null;
 224
 225         // by Area
 226         Pager<SearchResult<TaxonBase>>  pager = taxonService.findByDescriptionElementFullText(null, "Canada", null, subtree, null, null, false, null, null, null, null);
 227         Assert.assertEquals("Expecting one entity when searching for arae 'Canada'", 1, pager.getCount().intValue());
 228         // by Status
 229         pager = taxonService.findByDescriptionElementFullText(null, "present", null, subtree, null, null, false, null, null, null, null);
 230         Assert.assertEquals("Expecting one entity when searching for status 'present'", 1, pager.getCount().intValue());
 231     }
 232
 233     @SuppressWarnings("rawtypes")
 234     @Test
 235     @DataSet
 236     public final void testFindByDescriptionElementFullText_wildcard() throws IOException, LuceneParseException {
 237
 238         refreshLuceneIndex();
 239         TaxonNode subtree = null;
 240
 241         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, subtree, null, null, false, null, null, null, null);
 242         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1, pager.getCount().intValue());
 243     }
 244
 245     /**
 246      * Regression test for #3113 (hibernate search: wildcard query can cause BooleanQuery$TooManyClauses: maxClauseCount is set to 1024)
 247      *
 248      * @throws IOException
 249      * @throws LuceneParseException
 250      */
 251     @SuppressWarnings("rawtypes")
 252     @Test
 253     @DataSet
 254     public final void testFindByDescriptionElementFullText_TooManyClauses() throws IOException, LuceneParseException {
 255
 256         TaxonNode subtree = null;
 257
 258         // generate 1024 terms to reproduce the bug
 259         TaxonDescription description = (TaxonDescription) descriptionService.find(DESC_ABIES_ALBA_UUID);
 260         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 261         while(uniqueRandomStrs.size() < 1024){
 262             uniqueRandomStrs.add(RandomStringUtils.random(10, true, false));
 263         }
 264         for(String rndStr: uniqueRandomStrs){
 265             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 266         }
 267         descriptionService.saveOrUpdate(description);
 268         commitAndStartNewTransaction(null);
 269
 270         refreshLuceneIndex();
 271
 272         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, false, null, null, null, null);
 273         Assert.assertEquals("Expecting all 1024 entities grouped into one SearchResult item when searching for Rot*", 1, pager.getCount().intValue());
 274     }
 275
 276     /**
 277      * Regression test for #3116 (fulltext search: always only one page of results)
 278      *
 279      * @throws IOException
 280      * @throws LuceneParseException
 281      */
 282     @SuppressWarnings("rawtypes")
 283     @Test
 284     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 285     public final void testFullText_Paging() throws IOException, LuceneParseException {
 286
 287         TaxonNode subtree = null;
 288         Reference sec = ReferenceFactory.newDatabase();
 289         referenceService.save(sec);
 290
 291         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 292         int numOfItems = 100;
 293         while(uniqueRandomStrs.size() < numOfItems){
 294             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 295         }
 296
 297         for(String rndStr: uniqueRandomStrs){
 298
 299             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(Rank.SERIES()), sec);
 300             taxon.setTitleCache("Tax" + rndStr, true);
 301             taxonService.save(taxon);
 302
 303             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 304             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 305             descriptionService.saveOrUpdate(description);
 306         }
 307
 308         commitAndStartNewTransaction(new String[]{/*"TAXONBASE"*/});
 309         refreshLuceneIndex();
 310
 311         int pageSize = 10;
 312
 313         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, false, pageSize, null, null, null);
 314         Assert.assertEquals("unexpeted number of pages", Integer.valueOf(numOfItems / pageSize), pager.getPagesAvailable());
 315         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, false, pageSize, 9, null, null);
 316         Assert.assertNotNull("last page must have records", pager.getRecords());
 317         Assert.assertNotNull("last item on last page must exist", pager.getRecords().get(0));
 318         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, false, pageSize, 10, null, null);
 319         Assert.assertNotNull("last page + 1 must not have any records", pager.getRecords());
 320     }
 321
 322     /**
 323      * test for max score and sort by score of hit groups
 324      * with all matches per taxon in a single TextData  element
 325      * see {@link #testFullText_ScoreAndOrder_2()} for the complement
 326      * test with matches in multiple TextData per taxon
 327      *
 328      * @throws IOException
 329      * @throws LuceneParseException
 330      */
 331     @SuppressWarnings("rawtypes")
 332     @Test
 333     @DataSet
 334     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 335     public final void testFullText_ScoreAndOrder_1() throws IOException, LuceneParseException {
 336
 337         TaxonNode subtree = null;
 338         int numOfTaxa = 3;
 339
 340         UUID[] taxonUuids = new UUID[numOfTaxa];
 341         StringBuilder text = new StringBuilder();
 342
 343         for(int i = 0; i < numOfTaxa; i++){
 344
 345             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 346             taxon.setTitleCache("Taxon_" + i, true);
 347             taxonUuids[i] = taxon.getUuid();
 348             taxonService.save(taxon);
 349
 350             text.append(" ").append("Rot");
 351             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 352             description.addElement(TextData.NewInstance(text.toString(), Language.DEFAULT(), null));
 353             descriptionService.saveOrUpdate(description);
 354         }
 355
 356         commitAndStartNewTransaction(null);
 357         refreshLuceneIndex();
 358
 359         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, subtree, null, null, false, null, null, null, null);
 360         for(int i = 0; i < numOfTaxa; i++){
 361             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 362         }
 363         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 364     }
 365
 366     /**
 367      * test for max score and sort by score of hit groups
 368      * with all matches per taxon in a multiple TextData elements
 369      * see {@link #testFullText_ScoreAndOrder_1()} for the complement
 370      * test with matches in a single TextData per taxon
 371      *
 372      * @throws IOException
 373      * @throws LuceneParseException
 374      */
 375     @SuppressWarnings("rawtypes")
 376     @Test
 377     @DataSet
 378     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 379     public final void testFullText_ScoreAndOrder_2() throws IOException, LuceneParseException {
 380
 381         TaxonNode subtree = null;
 382         int numOfTaxa = 3;
 383
 384         UUID[] taxonUuids = new UUID[numOfTaxa];
 385
 386         for(int i = 0; i < numOfTaxa; i++){
 387
 388             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 389             taxon.setTitleCache("Taxon_" + i, true);
 390             taxonUuids[i] = taxon.getUuid();
 391             taxonService.save(taxon);
 392
 393             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 394             for(int k = 0; k < i; k++){
 395                 description.addElement(TextData.NewInstance("Rot", Language.DEFAULT(), null));
 396             }
 397             descriptionService.saveOrUpdate(description);
 398         }
 399
 400         commitAndStartNewTransaction(null);
 401         refreshLuceneIndex();
 402
 403         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, subtree, null, null, false, null, null, null, null);
 404         for(int i = 0; i < numOfTaxa; i++){
 405             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 406         }
 407         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 408     }
 409
 410
 411     /**
 412      * @throws IOException
 413      * @throws LuceneParseException
 414      * @throws LuceneMultiSearchException
 415      */
 416     @Test
 417     @DataSet
 418     public final void testFullText_Grouping() throws IOException, LuceneParseException, LuceneMultiSearchException {
 419
 420         TaxonNode subtree = null;
 421         TaxonDescription description = (TaxonDescription) descriptionService.find(DESC_ABIES_ALBA_UUID);
 422         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 423         int numOfItems = 100;
 424         while(uniqueRandomStrs.size() < numOfItems){
 425             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 426         }
 427         for(String rndStr: uniqueRandomStrs){
 428             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 429         }
 430         descriptionService.saveOrUpdate(description);
 431
 432         commitAndStartNewTransaction(new String[]{"DESCRIPTIONELEMENTBASE"});
 433
 434         refreshLuceneIndex();
 435
 436         int pageSize = 10;
 437
 438          boolean highlightFragments = true;
 439
 440         // test with findByDescriptionElementFullText
 441         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, highlightFragments, pageSize, null, null, null);
 442         logFreeTextSearchResults(pager, Level.DEBUG, null);
 443         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 444         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 445         Map<String, String[]> highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 446         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 447         int maxDocsPerGroup = 10;
 448         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 449
 450         // test with findByEverythingFullText
 451         pager = taxonService.findByEverythingFullText( "Rot*", null, subtree, includeUnpublished, null, highlightFragments, pageSize, null, null, null);
 452         logFreeTextSearchResults(pager, Level.DEBUG, null);
 453         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 454         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 455         highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 456         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 457         maxDocsPerGroup = 10;
 458         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 459
 460     }
 461
 462     @SuppressWarnings("rawtypes")
 463     @Test
 464     @DataSet
 465     public final void testFindByDescriptionElementFullText_TextData() throws IOException, LuceneParseException {
 466
 467         refreshLuceneIndex();
 468         TaxonNode subtree = null;
 469
 470         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, subtree, null, null, false, null, null, null, null);
 471         logFreeTextSearchResults(pager, Level.DEBUG, null);
 472         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 473         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 474         Assert.assertTrue("Expecting two docs, one for RUSSIAN and one for GERMAN", pager.getRecords().get(0).getDocs().size() == 2);
 475         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 476
 477
 478         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, subtree, null, null, false, null, null, null, null);
 479         Assert.assertEquals("Expecting one entity when searching for any type", 1, pager.getCount().intValue());
 480
 481         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, subtree, Arrays.asList(new Feature[]{Feature.UNKNOWN()}), null, false, null, null, null, null);
 482         Assert.assertEquals("Expecting one entity when searching for any type and for Feature DESCRIPTION", 1, pager.getCount().intValue());
 483
 484         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, subtree, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER()}), null, false, null, null, null, null);
 485         Assert.assertEquals("Expecting no entity when searching for any type and for Feature CHROMOSOME_NUMBER", 0, pager.getCount().intValue());
 486
 487         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, subtree, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER(), Feature.UNKNOWN()}), null, false, null, null, null, null);
 488         Assert.assertEquals("Expecting no entity when searching for any type and for Feature DESCRIPTION or CHROMOSOME_NUMBER", 1, pager.getCount().intValue());
 489
 490         pager = taxonService.findByDescriptionElementFullText(Distribution.class, "Abies", null, subtree, null, null, false, null, null, null, null);
 491         Assert.assertEquals("Expecting no entity when searching for Distribution", 0, pager.getCount().intValue());
 492
 493         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, subtree, null, Arrays.asList(new Language[]{}), false, null, null, null, null);
 494         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 495         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 496
 497         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, subtree, null, Arrays.asList(new Language[]{Language.RUSSIAN()}), false, null, null, null, null);
 498         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 499         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 500
 501         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 502         Assert.assertEquals("Expecting no entity", 0, pager.getCount().intValue());
 503
 504         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 505         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 506         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 507     }
 508
 509     @SuppressWarnings("rawtypes")
 510     @Test
 511     @DataSet
 512     public final void testFindByDescriptionElementFullText_MultipleWords() throws IOException, LuceneParseException {
 513
 514         refreshLuceneIndex();
 515         TaxonNode subtree = null;
 516
 517         // Pflanzenart aus der Gattung der Tannen
 518         long start = System.currentTimeMillis();
 519
 520         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, subtree, null, null, false, null, null, null, null);
 521         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 522
 523         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Wespen", null, subtree, null, null, false, null, null, null, null);
 524         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 525
 526         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, subtree, null, null, false, null, null, null, null);
 527         Assert.assertEquals("AND search : Expecting one entity", 1, pager.getCount().intValue());
 528
 529         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Wespen", null, subtree, null, null, false, null, null, null, null);
 530         Assert.assertEquals("AND search : Expecting no entity", 0, pager.getCount().intValue());
 531
 532         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, subtree, null, null, false, null, null, null, null);
 533         Assert.assertEquals("Phrase search : Expecting one entity", 1, pager.getCount().intValue());
 534
 535         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Wespen\"", null, subtree, null, null, false, null, null, null, null);
 536         Assert.assertEquals("Phrase search : Expecting one entity", 0, pager.getCount().intValue());
 537
 538         logger.info("testFindByDescriptionElementFullText_MultipleWords() duration: " + (System.currentTimeMillis() - start) + "ms");
 539
 540     }
 541
 542
 543     @SuppressWarnings("rawtypes")
 544     @Test
 545     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 546     public final void testFindByDescriptionElementFullText_modify_DescriptionElement() throws IOException, LuceneParseException {
 547
 548         refreshLuceneIndex();
 549         TaxonNode subtree = null;
 550
 551         //
 552         // modify the DescriptionElement
 553         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 554         Assert.assertTrue("Search did not return any results", pager.getRecords().size() > 0);
 555         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 556         Document indexDocument = pager.getRecords().get(0).getDocs().iterator().next();
 557         String[] descriptionElementUuidStr = indexDocument.getValues("uuid");
 558         String[] inDescriptionUuidStr = indexDocument.getValues("inDescription.uuid");
 559         // is only one uuid!
 560         DescriptionElementBase textData = descriptionService.getDescriptionElementByUuid(UUID.fromString(descriptionElementUuidStr[0]));
 561
 562         ((TextData)textData).removeText(Language.GERMAN());
 563         ((TextData)textData).putText(Language.SPANISH_CASTILIAN(), "abeto bals"+UTF8.SMALL_A_ACUTE+"mico");
 564
 565         descriptionService.saveDescriptionElement(textData);
 566         commitAndStartNewTransaction(null);
 567 //        printDataSet(System.out, new String[] {
 568 //                "DESCRIPTIONELEMENTBASE", "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING" }
 569 //        );
 570
 571         //
 572         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 573         Assert.assertEquals("The german 'Balsam-Tanne' TextData should no longer be indexed", 0, pager.getCount().intValue());
 574         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, subtree, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 575         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 576         pager = taxonService.findByDescriptionElementFullText(TextData.class, "bals"+UTF8.SMALL_A_ACUTE+"mico", null, subtree, null, null, false, null, null, null, null);
 577         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 578
 579         //
 580         // modify the DescriptionElement via the Description object
 581         DescriptionBase<?> description = descriptionService.find(UUID.fromString(inDescriptionUuidStr[0]));
 582         Set<DescriptionElementBase> elements = description.getElements();
 583         for( DescriptionElementBase elm : elements){
 584             if(elm.getUuid().toString().equals(descriptionElementUuidStr[0])){
 585                 ((TextData)elm).removeText(Language.SPANISH_CASTILIAN());
 586                 ((TextData)elm).putText(Language.POLISH(), "Jod"+UTF8.POLISH_L+"a balsamiczna");
 587             }
 588         }
 589         descriptionService.saveOrUpdate(description);
 590         commitAndStartNewTransaction(null);
 591         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, subtree, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 592         Assert.assertEquals("The spanish 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico' TextData should no longer be indexed", 0, pager.getCount().intValue());
 593         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsamiczna", null, subtree, null, Arrays.asList(new Language[]{Language.POLISH()}), false, null, null, null, null);
 594         Assert.assertEquals("expecting to find the POLISH 'Jod"+UTF8.POLISH_L+"a balsamiczna'", 1, pager.getCount().intValue());
 595     }
 596
 597     @SuppressWarnings("rawtypes")
 598     @Test
 599     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 600     public final void testFindByDescriptionElementFullText_modify_Taxon() throws IOException, LuceneParseException {
 601
 602         refreshLuceneIndex();
 603         TaxonNode subtree = null;
 604
 605         Taxon t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
 606         TaxonDescription d_abies_balsamea = (TaxonDescription)descriptionService.find(DESC_ABIES_BALSAMEA_UUID);
 607
 608         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne",
 609                 null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 610         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne'", 1, pager.getCount().intValue());
 611
 612         // exchange the Taxon with another one via the Taxon object
 613         // 1.) remove existing description:
 614         t_abies_balsamea.removeDescription(d_abies_balsamea);
 615
 616         taxonService.saveOrUpdate(t_abies_balsamea);
 617         commitAndStartNewTransaction(null);
 618
 619         t_abies_balsamea = (Taxon)taxonService.find(t_abies_balsamea.getUuid());
 620
 621         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne",
 622                 null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 623         Assert.assertEquals("'Balsam-Tanne' should no longer be found", 0, pager.getCount().intValue());
 624
 625         // 2.) create new description and add to taxon:
 626         TaxonDescription d_abies_balsamea_new = TaxonDescription.NewInstance();
 627         d_abies_balsamea_new
 628                 .addElement(TextData
 629                         .NewInstance(
 630                                 "Die Balsamtanne ist mit bis zu 30 m Höhe ein mittelgro"+UTF8.SHARP_S+"er Baum und kann bis zu 200 Jahre alt werden",
 631                                 Language.GERMAN(), null));
 632         t_abies_balsamea.addDescription(d_abies_balsamea_new);
 633         // set authorshipCache to null to avoid validation exception,
 634         // this is maybe not needed in future,  see ticket #3344
 635         IBotanicalName abies_balsamea = CdmBase.deproxy(t_abies_balsamea.getName(), TaxonName.class);
 636         abies_balsamea.setAuthorshipCache(null);
 637         printDataSet(System.err, new String[] {"LANGUAGESTRING_AUD"});
 638         taxonService.saveOrUpdate(t_abies_balsamea);
 639         commitAndStartNewTransaction(null);
 640
 641 //        printDataSet(System.out, new String[] {
 642 //                "DESCRIPTIONBASE"
 643 //        });
 644
 645         pager = taxonService.findByDescriptionElementFullText(TextData.class, "mittelgro"+UTF8.SHARP_S+"er Baum",
 646                 null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 647         Assert.assertEquals("the taxon should be found via the new Description", 1, pager.getCount().intValue());
 648     }
 649
 650     @SuppressWarnings("rawtypes")
 651     @Test
 652     @DataSet
 653     public final void testFindByDescriptionElementFullText_modify_Classification() throws IOException, LuceneParseException {
 654
 655         refreshLuceneIndex();
 656         TaxonNode subtree = null;
 657
 658         // put taxon into other classification, new taxon node
 659         Classification classification = classificationService.find(CLASSIFICATION_UUID);
 660         Classification alternateClassification = classificationService.find(CLASSIFICATION_ALT_UUID);
 661
 662         // TODO: why is the test failing when the childNode is already retrieved here, and not after the following four lines?
 663         //TaxonNode childNode = classification.getChildNodes().iterator().next();
 664
 665         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 666         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne' even if filtering by classification", 1, pager.getCount().intValue());
 667         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 668         Assert.assertEquals("GERMAN 'Balsam-Tanne' should NOT be found in other classification", 0, pager.getCount().intValue());
 669
 670         // check for the right taxon node
 671         TaxonNode childNode = classification.getChildNodes().iterator().next();
 672         Assert.assertEquals("expecting Abies balsamea sec.", childNode.getTaxon().getUuid(), ABIES_BALSAMEA_UUID);
 673         Assert.assertEquals("expecting default classification", childNode.getClassification().getUuid(), CLASSIFICATION_UUID);
 674
 675         // moving the taxon around, the rootnode is only a proxy
 676         alternateClassification.setRootNode(HibernateProxyHelper.deproxy(alternateClassification.getRootNode(), TaxonNode.class));
 677         alternateClassification.addChildNode(childNode, null, null);
 678
 679         classificationService.saveOrUpdate(alternateClassification);
 680         commitAndStartNewTransaction(null);
 681
 682 //        printDataSet(System.out, new String[] {
 683 //            "TAXONBASE", "TAXONNODE", "CLASSIFICATION"
 684 //        });
 685
 686         // reload classification
 687         classification = classificationService.find(CLASSIFICATION_UUID);
 688         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne",
 689                 alternateClassification, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 690         Assert.assertEquals("GERMAN 'Balsam-Tanne' should now be found in other classification", 1, pager.getCount().intValue());
 691
 692         classification.getChildNodes().clear();
 693         classificationService.saveOrUpdate(classification);
 694         commitAndStartNewTransaction(null);
 695
 696         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne",
 697                 classification, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 698         Assert.assertEquals("Now the GERMAN 'Balsam-Tanne' should NOT be found in original classification", 0, pager.getCount().intValue());
 699
 700     }
 701
 702     @SuppressWarnings("rawtypes")
 703     @Test
 704     @DataSet
 705     public final void testFindByDescriptionElementFullText_CategoricalData() throws IOException, LuceneParseException {
 706
 707         TaxonNode subtree = null;
 708         // add CategoricalData
 709         DescriptionBase d_abies_balsamea = descriptionService.find(DESC_ABIES_BALSAMEA_UUID);
 710         // Categorical data
 711         CategoricalData cdata = CategoricalData.NewInstance();
 712         cdata.setFeature(Feature.DESCRIPTION());
 713         State state = State.NewInstance("green", "green", "gn");
 714
 715         StateData statedata = StateData.NewInstance(state);
 716         statedata.putModifyingText(Language.ENGLISH(), "always, even during winter");
 717         cdata.addStateData(statedata);
 718         d_abies_balsamea.addElement(cdata);
 719
 720         UUID termUUID = termService.save(state).getUuid();
 721         descriptionService.save(d_abies_balsamea);
 722
 723         commitAndStartNewTransaction(null);
 724
 725 //        printDataSet(System.out, new String[] {
 726 //                 "STATEDATA", "STATEDATA_DEFINEDTERMBASE", "STATEDATA_LANGUAGESTRING", "LANGUAGESTRING"});
 727
 728         refreshLuceneIndex();
 729
 730         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CategoricalData.class, "green", null, subtree, null, null, false, null, null, null, null);
 731         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 732         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 733         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 734         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 735
 736
 737         //TODO modify the StateData
 738         TaxonBase taxon = pager.getRecords().get(0).getEntity();
 739
 740         String newName = "Quercus robur";
 741         taxon.setTitleCache(newName + " sec. ", true);
 742
 743         taxonService.saveOrUpdate(taxon);
 744         commitAndStartNewTransaction(null);
 745
 746         taxon = taxonService.find(taxon.getUuid());
 747         Assert.assertEquals(newName + " sec. ", taxon.getTitleCache());
 748         DefinedTermBase term = termService.find(termUUID);
 749
 750         termService.delete(term);
 751
 752     }
 753
 754     @SuppressWarnings("rawtypes")
 755     @Test
 756     @DataSet
 757     public final void testFindByDescriptionElementFullText_Highlighting() throws IOException, LuceneParseException {
 758
 759         refreshLuceneIndex();
 760         TaxonNode subtree = null;
 761
 762         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, subtree, null, null, true, null, null, null, null);
 763         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 764         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 765         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
 766         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 767         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Abies</B>"));
 768
 769         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, subtree, null, null, true, null, null, null, null);
 770         searchResult = pager.getRecords().get(0);
 771         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 772         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 773         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") || fragments[0].contains("<B>Tannen</B>"));
 774
 775         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, subtree, null, null, true, null, null, null, null);
 776         searchResult = pager.getRecords().get(0);
 777         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 778         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 779         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") && fragments[0].contains("<B>Tannen</B>"));
 780
 781         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, subtree, null, null, true, null, null, null, null);
 782         searchResult = pager.getRecords().get(0);
 783         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 784         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 785         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B> <B>aus</B> <B>der</B> <B>Gattung</B> <B>der</B> <B>Tannen</B>"));
 786
 787         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Gatt*", null, subtree, null, null, true, null, null, null, null);
 788         searchResult = pager.getRecords().get(0);
 789         Assert.assertTrue("Wildcard search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 790         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 791         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Gatt"));
 792     }
 793
 794
 795     @Test
 796     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 797     public final void testFindByFullText() throws IOException, LuceneParseException {
 798
 799         refreshLuceneIndex();
 800
 801         classificationService.find(CLASSIFICATION_UUID);
 802         TaxonNode subtree = null;
 803
 804         boolean NO_UNPUBLISHED = false;
 805
 806         Pager<SearchResult<TaxonBase>> pager = taxonService.findByFullText(null, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 7
 807 //        logFreeTextSearchResults(pager, Level.DEBUG, null);
 808         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 809
 810         //subtree
 811         subtree = nodeService.find(ROOTNODE_CLASSIFICATION_5000);
 812         pager = taxonService.findByFullText(null, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 0
 813         Assert.assertEquals("Expecting 2 entities", 2, pager.getCount().intValue());
 814         subtree = null;
 815
 816         pager = taxonService.findByFullText(null, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 817 //        logFreeTextSearchResults(pager, Level.DEBUG, null);
 818         Assert.assertEquals("Expecting 6 entities", 6, pager.getCount().intValue());
 819         Synonym abiesSubalpina = (Synonym)taxonService.find(ABIES_SUBALPINA_UUID);
 820
 821         //accepted published, syn not published
 822         abiesSubalpina.getAcceptedTaxon().setPublish(true);
 823         commitAndStartNewTransaction();
 824         pager = taxonService.findByFullText(null, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 825         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 826
 827         //accepted published, syn published
 828         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 829         abiesSubalpina.setPublish(true);
 830         commitAndStartNewTransaction();
 831         pager = taxonService.findByFullText(null, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 832         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 833
 834         //accepted not published, syn published
 835         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 836         abiesSubalpina.getAcceptedTaxon().setPublish(false);
 837         commitAndStartNewTransaction();
 838         pager = taxonService.findByFullText(null, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 839         Assert.assertEquals("Expecting 6 entities. Synonym and accepted should not be found, though synonym is published",
 840                 6, pager.getCount().intValue());
 841
 842         pager = taxonService.findByFullText(Taxon.class, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 6
 843         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 844
 845         pager = taxonService.findByFullText(Synonym.class, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 1
 846         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 847         pager = taxonService.findByFullText(Synonym.class, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 1
 848         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 849
 850         pager = taxonService.findByFullText(TaxonBase.class, "sec", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 7
 851         Assert.assertEquals("Expecting 8 entities", 9, pager.getCount().intValue());
 852
 853         pager = taxonService.findByFullText(null, "genus", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 1
 854         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 855
 856         pager = taxonService.findByFullText(Taxon.class, "subalpina", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 0
 857         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 858
 859
 860         // synonym in classification ???
 861     }
 862
 863     @Test
 864     @DataSet
 865     public final void testPrepareByAreaSearch() throws IOException, LuceneParseException {
 866
 867         List<PresenceAbsenceTerm> statusFilter = new ArrayList<>();
 868         List<NamedArea> areaFilter = new ArrayList<>();
 869         areaFilter.add(germany);
 870         areaFilter.add(canada);
 871         areaFilter.add(russia);
 872
 873         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDistribution(areaFilter, statusFilter, null, null, 20, 0, null, null);
 874         Assert.assertEquals("Expecting 2 entities", Integer.valueOf(2), Integer.valueOf(pager.getRecords().size()));
 875
 876     }
 877
 878     @Test
 879     @DataSet
 880     public final void testFindTaxaAndNamesByFullText_synonymClassificationSubtree() throws IOException, LuceneParseException, LuceneMultiSearchException {
 881
 882         refreshLuceneIndex();
 883         Classification classification = null;
 884         TaxonNode subtree = null;
 885
 886         //
 887         Pager<SearchResult<TaxonBase>> pager;
 888         EnumSet<TaxaAndNamesSearchMode> taxaAndSynonyms = EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished);
 889         pager = taxonService.findTaxaAndNamesByFullText(
 890                 taxaAndSynonyms, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 891         Assert.assertEquals("doTaxa & doSynonyms & unpublished", 8, pager.getCount().intValue());
 892
 893         EnumSet<TaxaAndNamesSearchMode> taxaOnly = EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished);
 894
 895         //classification
 896         classification = classificationService.find(CLASSIFICATION_UUID);
 897         pager = taxonService.findTaxaAndNamesByFullText(
 898                 taxaAndSynonyms, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 899         Assert.assertEquals("doTaxa & doSynonyms & unpublished", 2, pager.getCount().intValue());
 900             //taxa only
 901         pager = taxonService.findTaxaAndNamesByFullText(
 902                 taxaOnly, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 903         Assert.assertEquals("doTaxa & unpublished", 1, pager.getCount().intValue());
 904             //synonyms only
 905         pager = taxonService.findTaxaAndNamesByFullText(
 906                 taxaOnly, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 907         Assert.assertEquals("doSynonyms & unpublished", 1, pager.getCount().intValue());
 908
 909         classification = null;
 910
 911         //subtree
 912        subtree = nodeService.find(ROOTNODE_CLASSIFICATION_5000);
 913        pager = taxonService.findTaxaAndNamesByFullText(
 914                taxaAndSynonyms, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 915        Assert.assertEquals("doTaxa & doSynonyms & unpublished", 2, pager.getCount().intValue());
 916          //taxa only
 917        pager = taxonService.findTaxaAndNamesByFullText(
 918                taxaOnly, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 919        Assert.assertEquals("doTaxa & unpublished", 1, pager.getCount().intValue());
 920        subtree = null;
 921
 922     }
 923
 924     @Test
 925     @DataSet
 926     public final void testFindTaxaAndNamesByFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
 927
 928         refreshLuceneIndex();
 929         TaxonNode subtree = null;
 930
 931         Classification alternateClassification = classificationService.find(CLASSIFICATION_ALT_UUID);
 932         Synonym abiesSubalpina = (Synonym)taxonService.find(ABIES_SUBALPINA_UUID);
 933
 934         Pager<SearchResult<TaxonBase>> pager;
 935         EnumSet<TaxaAndNamesSearchMode> modes = EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished);
 936         pager = taxonService.findTaxaAndNamesByFullText(
 937                 modes, "Abies", null, subtree, null, null, null, true, null, null, null, null);
 938         Assert.assertEquals("doTaxa & doSynonyms & unpublished", 8, pager.getCount().intValue());
 939 //      logPagerRecords(pager, Level.DEBUG);
 940
 941          //unpublished
 942         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 943                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 944         Assert.assertEquals("doTaxa & doSynonyms, published only", 6, pager.getCount().intValue());
 945
 946         //accepted published, syn not published
 947         abiesSubalpina.getAcceptedTaxon().setPublish(true);
 948         commitAndStartNewTransaction();
 949         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 950                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 951         Assert.assertEquals("doTaxa & doSynonyms, accepted published", 7, pager.getCount().intValue());
 952
 953         //accepted published, syn published
 954         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 955         abiesSubalpina.setPublish(true);
 956         commitAndStartNewTransaction();
 957         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 958                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 959         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 960
 961         //accepted not published, syn published
 962         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 963         abiesSubalpina.getAcceptedTaxon().setPublish(false);
 964         commitAndStartNewTransaction();
 965         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 966                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 967         Assert.assertEquals("Expecting 6 entities. Synonym and accepted should not be found, though synonym is published",
 968                 6, pager.getCount().intValue());
 969
 970         EnumSet<TaxaAndNamesSearchMode> searchMode = EnumSet.allOf(TaxaAndNamesSearchMode.class);
 971         pager = taxonService.findTaxaAndNamesByFullText(
 972                 searchMode, "Abies", null, subtree, null, null, null, true, null, null, null, null);
 973 //        logPagerRecords(pager, Level.DEBUG);
 974         Assert.assertEquals("all search modes", 8, pager.getCount().intValue());
 975         searchMode.remove(TaxaAndNamesSearchMode.includeUnpublished);
 976         pager = taxonService.findTaxaAndNamesByFullText(
 977                 searchMode, "Abies", null, subtree, null, null, null, true, null, null, null, null);
 978         Assert.assertEquals("all search modes except unpublished", 6, pager.getCount().intValue());
 979
 980         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.allOf(TaxaAndNamesSearchMode.class),
 981                 "Abies", alternateClassification, subtree, null, null, null, true, null, null, null, null);
 982 //        logPagerRecords(pager, Level.DEBUG);
 983         Assert.assertEquals("all search modes, filtered by alternateClassification", 1, pager.getCount().intValue());
 984
 985         pager = taxonService.findTaxaAndNamesByFullText(
 986                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 987                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 988         Assert.assertEquals("Expecting 2 entity", 2, pager.getCount().intValue());
 989         Set<UUID> uuids = getTaxonUuidSet(pager);
 990         Assert.assertTrue("The real synonym should be contained", uuids.contains(ABIES_SUBALPINA_UUID));
 991         Assert.assertTrue("The pro parte synonym should be contained",uuids.contains(ABIES_LASIOCARPA_UUID));
 992         //without published
 993         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
 994                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 995         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 996
 997
 998         pager = taxonService.findTaxaAndNamesByFullText(
 999                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
1000                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
1001         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
1002
1003         pager = taxonService.findTaxaAndNamesByFullText(
1004                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
1005                 "Tanne", null, subtree, null, null, null, true, null, null, null, null);
1006         Assert.assertEquals("Expecting 1 entity", 1, pager.getRecords().size());
1007         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1008         pager = taxonService.findTaxaAndNamesByFullText(
1009                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
1010                 "Tanne", null, subtree, null, null, null, true, null, null, null, null);
1011         Assert.assertEquals("Expecting 0 entity", 0, pager.getRecords().size());
1012
1013         //misapplied names
1014         pager = taxonService.findTaxaAndNamesByFullText(
1015                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1016                 "kawakamii", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1017         logFreeTextSearchResults(pager, Level.DEBUG, null);
1018         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1019         //unpublish accepted taxon
1020         pager = taxonService.findTaxaAndNamesByFullText(
1021                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1022                 "kawakamii", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1023         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
1024         //published accepted taxon/misapplied name
1025         Taxon abiesBalsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1026         abiesBalsamea.setPublish(true);
1027         commitAndStartNewTransaction();
1028         pager = taxonService.findTaxaAndNamesByFullText(
1029                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1030                 "kawakamii", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1031         Assert.assertEquals("Expecting 1 entities", 1, pager.getCount().intValue());
1032         //unpublished misapplied name
1033         Taxon misapplied = (Taxon)taxonService.find(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1034         misapplied.setPublish(false);
1035         commitAndStartNewTransaction();
1036         pager = taxonService.findTaxaAndNamesByFullText(
1037                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1038                 "kawakamii", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1039         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
1040
1041     }
1042
1043     @Test
1044     @DataSet
1045     public final void testFindTaxaAndNamesByFullText_wildcard() throws IOException, LuceneParseException, LuceneMultiSearchException {
1046
1047         refreshLuceneIndex();
1048         TaxonNode subtree = null;
1049
1050         Pager<SearchResult<TaxonBase>> pager;
1051          pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1052                 "Abi*", null, subtree, null, null, null, true, null, null, null, null);
1053 //      logFreeTextSearchResults(pager, Level.DEBUG, null);
1054         Assert.assertEquals("doTaxa & doSynonyms published only", 6, pager.getCount().intValue());
1055         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1056                 "*bies", null, subtree, null, null, null, true, null, null, null, null);
1057         Assert.assertEquals("doTaxa & doSynonyms, published only", 6, pager.getCount().intValue());
1058         // logFreeTextSearchResults(pager, Level.ERROR, null);
1059         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1060                 "?bies", null, subtree, null, null, null, true, null, null, null, null);
1061         Assert.assertEquals("doTaxa & doSynonyms, published only", 6, pager.getCount().intValue());
1062         // logFreeTextSearchResults(pager, Level.ERROR, null);
1063         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1064                 "*", null, subtree, null, null, null, true, null, null, null, null);
1065         Assert.assertEquals("doTaxa & doSynonyms, published only", 7, pager.getCount().intValue());
1066     }
1067
1068     @Test
1069     @DataSet
1070     // @Ignore // FIXME: fails due  org.apache.lucene.queryparser.classic.ParseException: Cannot parse 'relatedFrom.titleCache:()': Encountered " ")" ") "" at line 1, column 24.
1071     public final void testFindTaxaAndNamesByFullText_empty_querString() throws IOException, LuceneParseException, LuceneMultiSearchException {
1072
1073         refreshLuceneIndex();
1074         TaxonNode subtree = null;
1075
1076         Pager<SearchResult<TaxonBase>> pager;
1077         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
1078                 "", null, subtree, null, null, null, true, null, null, null, null);
1079         Assert.assertEquals("doTaxa, published only", 7, pager.getCount().intValue());
1080
1081         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1082                 "", null, subtree, null, null, null, true, null, null, null, null);
1083         Assert.assertEquals("doTaxa & doSynonyms, published only", 7, pager.getCount().intValue());
1084
1085         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doMisappliedNames),
1086                 null, null, subtree, null, null, null, true, null, null, null, null);
1087         Assert.assertEquals("doTaxa & doMisappliedNames published only", 7, pager.getCount().intValue());
1088
1089         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.doTaxaByCommonNames),
1090                 null, null, subtree, null, null, null, true, null, null, null, null);
1091         Assert.assertEquals("doTaxa & doMisappliedNames & doTaxaByCommonNames , published only", 7, pager.getCount().intValue());
1092         // logFreeTextSearchResults(pager, Level.ERROR, null);
1093     }
1094
1095     @Test
1096     @DataSet
1097     //test for https://dev.e-taxonomy.eu/redmine/issues/7486
1098     public final void testFindTaxaAndNamesByFullText_synonymsAndMisapplied_7486() throws IOException, LuceneParseException, LuceneMultiSearchException {
1099
1100         refreshLuceneIndex();
1101         TaxonNode subtree = null;
1102
1103         //misapplied names
1104         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1105                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1106                 "Abies", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1107         logFreeTextSearchResults(pager, Level.DEBUG, null);
1108         Assert.assertEquals("Expecting 3 entity", 3, pager.getCount().intValue());
1109         Set<UUID> uuids = getTaxonUuidSet(pager);
1110         Assert.assertTrue("The real synonym should be contained", uuids.contains(ABIES_SUBALPINA_UUID));
1111         Assert.assertTrue("The pro parte synonym should be contained",uuids.contains(ABIES_LASIOCARPA_UUID));
1112         Assert.assertTrue("The misapplied name should be contained",uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1113     }
1114
1115     @Test
1116     @DataSet
1117     public final void testFindTaxaAndNamesByFullText_PhraseQuery() throws IOException, LuceneParseException, LuceneMultiSearchException {
1118
1119         refreshLuceneIndex();
1120         TaxonNode subtree = null;
1121
1122         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1123                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1124                 "\"Abies alba\"", null, subtree, null, null, null, true, null, null, null, null);
1125 //        logPagerRecords(pager, Level.DEBUG);
1126         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 1, pager.getCount().intValue());
1127
1128         pager = taxonService.findTaxaAndNamesByFullText(
1129                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1130                 "\"Abies al*\"", null, subtree, null, null, null, true, null, null, null, null);
1131 //        logPagerRecords(pager, Level.DEBUG);
1132         Assert.assertEquals("doTaxa & doSynonyms with complex phrase query", 1, pager.getCount().intValue());
1133
1134         pager = taxonService.findTaxaAndNamesByFullText(
1135                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1136                 "\"Abies*\"", null, subtree, null, null, null, true, null, null, null, null);
1137 //        logPagerRecords(pager, Level.DEBUG);
1138         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 8, pager.getCount().intValue());
1139
1140         pager = taxonService.findTaxaAndNamesByFullText(
1141                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1142                 "\"Abies*\"", null, subtree, null, null, null, true, null, null, null, null);
1143 //        logPagerRecords(pager, Level.DEBUG);
1144         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 8, pager.getCount().intValue());
1145
1146     }
1147
1148     @Test
1149     @DataSet
1150     public final void testFindTaxaAndNamesByFullText_Sort() throws IOException, LuceneParseException, LuceneMultiSearchException {
1151
1152         refreshLuceneIndex();
1153         TaxonNode subtree = null;
1154
1155         List<OrderHint> orderHints = new ArrayList<>();
1156
1157 //        String[] docFields2log = new String[]{"id"};
1158
1159         // SortById
1160         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
1161         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1162                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
1163                 "Abies", null, subtree, null, null, null, true, null, null, orderHints, null);
1164 //        logSearchResults(pager, Level.DEBUG, docFields2log);
1165         int lastId = -1;
1166         for(SearchResult<TaxonBase> rs : pager.getRecords()){
1167             if(lastId != -1){
1168                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
1169             }
1170             lastId = rs.getEntity().getId();
1171         }
1172
1173         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
1174         pager = taxonService.findTaxaAndNamesByFullText(
1175                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1176                 "Abies", null, subtree, null, null, null, true, null, null, orderHints, null);
1177 //        logSearchResults(pager, Level.DEBUG, docFields2log);
1178
1179         lastId = -1;
1180         for(SearchResult<TaxonBase> rs : pager.getRecords()){
1181             if(lastId != -1){
1182                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
1183             }
1184             lastId = rs.getEntity().getId();
1185         }
1186
1187         // Sortby NOMENCLATURAL_SORT_ORDER TODO make assertions !!!
1188         orderHints.clear();
1189         orderHints.addAll(OrderHint.NOMENCLATURAL_SORT_ORDER.asList());
1190         pager = taxonService.findTaxaAndNamesByFullText(
1191                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1192                 "Abies", null, subtree, null, null, null, true, null, null, orderHints, null);
1193         logFreeTextSearchResults(pager, Level.DEBUG, null);
1194
1195     }
1196
1197     @Test
1198     @DataSet
1199     @Ignore //ignore until #7487 is fixed
1200     public final void testFindTaxaAndNamesByFullText_AreaFilter_7487() throws IOException, LuceneParseException, LuceneMultiSearchException {
1201         refreshLuceneIndex();
1202         TaxonNode subtree = null;
1203         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1204         a_germany_canada_russia.add(germany);
1205         a_germany_canada_russia.add(canada);
1206         a_germany_canada_russia.add(russia);
1207
1208         Set<PresenceAbsenceTerm> present_native = new HashSet<>();
1209         present_native.add(PresenceAbsenceTerm.PRESENT());
1210         present_native.add(PresenceAbsenceTerm.NATIVE());
1211
1212         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1213                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1214                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1215         Assert.assertEquals("Synonyms with matching area filter", 2, pager.getCount().intValue());
1216         Set<UUID> uuids = this.getTaxonUuidSet(pager);
1217         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1218         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1219
1220         //pro parte syn => partial syn
1221         Taxon t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1222         Set<TaxonRelationship> relsTo = t_abies_balsamea.getProParteAndPartialSynonymRelations();
1223         Assert.assertEquals(1, relsTo.size());
1224         TaxonRelationship taxonRelation = relsTo.iterator().next();
1225         taxonRelation.setType(TaxonRelationshipType.PARTIAL_SYNONYM_FOR());
1226         taxonService.saveOrUpdate(t_abies_balsamea);
1227         commitAndStartNewTransaction(null);
1228
1229         //should give same results as above
1230         pager = taxonService.findTaxaAndNamesByFullText(
1231                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1232                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1233 //        Assert.assertEquals("Synonyms with matching area filter", 2, pager.getCount().intValue());
1234 //        uuids = this.getTaxonUuidSet(pager);
1235 //        Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1236 //        Assert.assertTrue("Partial synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1237
1238         ///MISAPPLIED
1239         pager = taxonService.findTaxaAndNamesByFullText(
1240                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished),
1241                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1242         Assert.assertEquals("misappliedNames with matching area & status filter", 3, pager.getCount().intValue());
1243         uuids = this.getTaxonUuidSet(pager);
1244         Assert.assertTrue("Misapplied name should be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1245
1246         t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1247         relsTo = t_abies_balsamea.getMisappliedNameRelations();
1248         Assert.assertEquals(1, relsTo.size());
1249         taxonRelation = relsTo.iterator().next();
1250         Assert.assertEquals(taxonRelation.getFromTaxon().getUuid(), DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1251         taxonRelation.setType(TaxonRelationshipType.PRO_PARTE_MISAPPLIED_NAME_FOR());
1252         taxonService.saveOrUpdate(t_abies_balsamea);
1253         commitAndStartNewTransaction(null);
1254
1255         //strange it works here before fixing #7487 already
1256         pager = taxonService.findTaxaAndNamesByFullText(
1257                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished),
1258                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1259         Assert.assertEquals("misappliedNames with matching area & status filter", 3, pager.getCount().intValue());
1260         uuids = this.getTaxonUuidSet(pager);
1261         Assert.assertTrue("Pro parte misapplied name should be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1262
1263     }
1264
1265
1266     @Test
1267     @DataSet
1268     public final void testFindTaxaAndNamesByFullText_AreaFilter() throws IOException, LuceneParseException, LuceneMultiSearchException {
1269
1270         refreshLuceneIndex();
1271         TaxonNode subtree = null;
1272
1273         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1274         a_germany_canada_russia.add(germany);
1275         a_germany_canada_russia.add(canada);
1276         a_germany_canada_russia.add(russia);
1277
1278         Set<NamedArea> a_russia = new HashSet<>();
1279         a_russia.add(russia);
1280
1281         Set<PresenceAbsenceTerm> present = new HashSet<>();
1282         present.add(PresenceAbsenceTerm.PRESENT());
1283
1284         Set<PresenceAbsenceTerm> present_native = new HashSet<>();
1285         present_native.add(PresenceAbsenceTerm.PRESENT());
1286         present_native.add(PresenceAbsenceTerm.NATIVE());
1287
1288         Set<PresenceAbsenceTerm> absent = new HashSet<>();
1289         absent.add(PresenceAbsenceTerm.ABSENT());
1290
1291         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1292                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished),
1293                 "Abies", null, subtree, a_germany_canada_russia, null, null, true, null, null, null, null);
1294         logFreeTextSearchResults(pager, Level.DEBUG, null);
1295         Assert.assertEquals("Synonyms with matching area filter", 2, pager.getCount().intValue());
1296         Set<UUID> uuids = this.getTaxonUuidSet(pager);
1297         Assert.assertTrue("Abies alba (accepted with distribution) should be in", uuids.contains(ABIES_ALBA_UUID));
1298         Assert.assertTrue("Abies balsamea (accepted with distribution) should be in", uuids.contains(ABIES_BALSAMEA_UUID));
1299
1300         pager = taxonService.findTaxaAndNamesByFullText(
1301                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1302                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1303         Assert.assertEquals("Synonyms with matching area filter", 2, pager.getCount().intValue());
1304         uuids = this.getTaxonUuidSet(pager);
1305         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1306         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1307
1308         pager = taxonService.findTaxaAndNamesByFullText(
1309                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1310                 "Abies", null, subtree, a_germany_canada_russia, null, null, true, null, null, null, null);
1311         logFreeTextSearchResults(pager, Level.DEBUG, null);
1312         Assert.assertEquals("taxa and synonyms with matching area filter", 4, pager.getCount().intValue());
1313         uuids = this.getTaxonUuidSet(pager);
1314         Assert.assertTrue("Accepted taxon with area should be in", uuids.contains(ABIES_ALBA_UUID));
1315         Assert.assertTrue("Accepted taxon with area should be in", uuids.contains(ABIES_BALSAMEA_UUID));
1316         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1317         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1318         Assert.assertFalse("Misapplied name should NOT be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1319
1320         pager = taxonService.findTaxaAndNamesByFullText(
1321                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1322                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1323         Assert.assertEquals("taxa and synonyms with matching area & status filter 4", 4, pager.getCount().intValue());
1324         uuids = this.getTaxonUuidSet(pager);
1325         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1326         Assert.assertTrue("Accepted taxon with area should be in", uuids.contains(ABIES_ALBA_UUID));
1327         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1328         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1329
1330         pager = taxonService.findTaxaAndNamesByFullText(
1331                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1332                 "Abies", null, subtree, a_germany_canada_russia, present, null, true, null, null, null, null);
1333         Assert.assertEquals("taxa and synonyms with matching area & status filter 3", 3, pager.getCount().intValue());
1334         uuids = this.getTaxonUuidSet(pager);
1335         Assert.assertTrue("Abies balsamea (accepted taxon) should be in", uuids.contains(ABIES_BALSAMEA_UUID));
1336         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1337         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1338
1339         pager = taxonService.findTaxaAndNamesByFullText(
1340                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1341                 "Abies", null, subtree, a_russia, present, null, true, null, null, null, null);
1342         Assert.assertEquals("taxa and synonyms with non matching area & status filter", 0, pager.getCount().intValue());
1343
1344         pager = taxonService.findTaxaAndNamesByFullText(
1345                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
1346                 "Tanne", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1347         Assert.assertEquals("ByCommonNames with area filter", 1, pager.getCount().intValue());
1348         uuids = this.getTaxonUuidSet(pager);
1349         Assert.assertTrue("Abies balsamea should be in", uuids.contains(ABIES_BALSAMEA_UUID));
1350
1351         // abies_kawakamii_sensu_komarov as misapplied name for t_abies_balsamea
1352         pager = taxonService.findTaxaAndNamesByFullText(
1353                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1354                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1355         Assert.assertEquals("misappliedNames with matching area & status filter", 1, pager.getCount().intValue());
1356         uuids = this.getTaxonUuidSet(pager);
1357         Assert.assertTrue("Misapplied name should  be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1358
1359
1360         // 1. remove existing taxon relation
1361         Taxon t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1362         Set<TaxonRelationship> relsTo = t_abies_balsamea.getMisappliedNameRelations();
1363         Assert.assertEquals(1, relsTo.size());
1364         TaxonRelationship taxonRelation = relsTo.iterator().next();
1365         t_abies_balsamea.removeTaxonRelation(taxonRelation);
1366         taxonService.saveOrUpdate(t_abies_balsamea);
1367         commitAndStartNewTransaction(null);
1368
1369         pager = taxonService.findTaxaAndNamesByFullText(
1370                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1371                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1372         Assert.assertEquals("misappliedNames with matching area & status filter, should match nothing now", 0, pager.getCount().intValue());
1373
1374         // 2. now add abies_kawakamii_sensu_komarov as misapplied name for t_abies_alba and search for misapplications in Russia: ABSENT
1375         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1376         Taxon t_abies_alba = (Taxon)taxonService.find(ABIES_ALBA_UUID);
1377         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1378         taxonService.update(t_abies_kawakamii_sensu_komarov);
1379         commitAndStartNewTransaction(null);
1380
1381         pager = taxonService.findTaxaAndNamesByFullText(
1382                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1383                 "Abies", null, subtree, a_germany_canada_russia, absent, null, true, null, null, null, null);
1384         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1385         uuids = this.getTaxonUuidSet(pager);
1386         Assert.assertTrue("Misapplied name should  be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1387
1388     }
1389
1390     @Test
1391     @DataSet
1392     //https://dev.e-taxonomy.eu/redmine/issues/5477
1393     public final void testFindTaxaAndNamesByFullText_AreaFilter_issue5477() throws IOException, LuceneParseException, LuceneMultiSearchException {
1394
1395         TaxonNode subtree = null;
1396         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1397         a_germany_canada_russia.add(germany);
1398         a_germany_canada_russia.add(canada);
1399         a_germany_canada_russia.add(russia);
1400
1401
1402         Set<PresenceAbsenceTerm> absent = new HashSet<>();
1403         absent.add(PresenceAbsenceTerm.ABSENT());
1404
1405         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1406         Taxon t_abies_alba = (Taxon)taxonService.find(ABIES_ALBA_UUID);
1407         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1408
1409         /* Since the upgrade from hibernate search 4 to 5.5
1410          * triggering an update of t_abies_alba is no longer sufficient to also update the
1411          * document of t_abies_kawakamii_sensu_komarov in the lucene index.
1412          * the last test in testFindTaxaAndNamesByFullText_AreaFilter() failed in this case.
1413          * This situation is reproduced here:
1414          */
1415         taxonService.update(t_abies_alba);
1416
1417         commitAndStartNewTransaction(null);
1418
1419         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1420                   EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1421                   "Abies", null, subtree, a_germany_canada_russia, absent, null, true, null, null, null, null);
1422         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1423     }
1424
1425
1426     /**
1427      * Regression test for #3119: fulltext search: Entity always null whatever search
1428      *
1429      * @throws IOException
1430      * @throws LuceneParseException
1431      * @throws LuceneMultiSearchException
1432      */
1433     @Test
1434     @DataSet
1435     public final void testFindByEverythingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1436
1437         refreshLuceneIndex();
1438         TaxonNode subtree = null;
1439         EnumSet<TaxaAndNamesSearchMode> mode = TaxaAndNamesSearchMode.taxaAndSynonymsWithUnpublished();
1440         // via Taxon
1441         Pager<SearchResult<TaxonBase>>pager = taxonService.findByEverythingFullText("Abies", null, subtree, includeUnpublished, null, true, null, null, null, null);
1442 //        Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(mode,
1443 //                "Abies", null, null, null, null, true, null, null, null, null);
1444         logFreeTextSearchResults(pager, Level.DEBUG, null);
1445         Assert.assertTrue("Expecting at least 7 entities for 'Abies'", pager.getCount() > 7);
1446         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1447 //        Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1448
1449         // via DescriptionElement
1450         pager = taxonService.findByEverythingFullText("present", null, subtree, includeUnpublished, null, true, null, null, null, null);
1451         //this is not covered by findTaxaAndNamesByFullText
1452 //        pager = taxonService.findTaxaAndNamesByFullText(mode,
1453 //                "present", null, null, null, null, true, null, null, null, null);
1454         Assert.assertEquals("Expecting one entity when searching for area 'present'", 1, pager.getCount().intValue());
1455         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1456         Assert.assertEquals("Expecting Taxon entity", Taxon.class, CdmBase.deproxy(pager.getRecords().get(0).getEntity()).getClass());
1457         Assert.assertEquals("Expecting Taxon ", ABIES_BALSAMEA_UUID, pager.getRecords().get(0).getEntity().getUuid());
1458
1459     }
1460
1461
1462     @Test
1463     @DataSet
1464     public final void findByEveryThingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1465
1466         refreshLuceneIndex();
1467         TaxonNode subtree = null;
1468
1469         Classification classification = null;
1470         EnumSet<TaxaAndNamesSearchMode> mode = TaxaAndNamesSearchMode.taxaAndSynonymsWithUnpublished();
1471
1472         Pager<SearchResult<TaxonBase>> pager = taxonService.findByEverythingFullText("genus", null, subtree, includeUnpublished, null, false, null, null, null, null); // --> 1
1473 //        Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(mode,
1474 //                "genus", classification, null, null, null, false, null, null, null, null);
1475         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1476
1477         //FIXME FAILS: abies balamea is returned twice, see also testFullText_Grouping()
1478         pager = taxonService.findByEverythingFullText("Balsam", null, subtree, includeUnpublished, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1479         logFreeTextSearchResults(pager, Level.DEBUG, null);
1480 //        pager = taxonService.findTaxaAndNamesByFullText(EnumSet.allOf(TaxaAndNamesSearchMode.class),
1481 //                "Balsam", classification, null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1482         Assert.assertEquals("expecting to find the Abies balsamea via the GERMAN DescriptionElements", 1, pager.getCount().intValue());
1483
1484         //TODO fieldHighlight does not yet work
1485         pager = taxonService.findByEverythingFullText("Abies", null, subtree, includeUnpublished, null, true, null, null, null, null);
1486 //        pager = taxonService.findTaxaAndNamesByFullText(mode,
1487 //                "Abies", classification, null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1488         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
1489         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
1490         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
1491         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
1492         Assert.assertTrue("first fragments should contains serch term", fragments[0].toLowerCase().contains("<b>abies</b>"));
1493     }
1494
1495 //    @SuppressWarnings("rawtypes")
1496 //    @Test
1497 //    @DataSet
1498 //    public final void benchmarkFindTaxaAndNamesHql() throws IOException, LuceneParseException {
1499 //
1500 //        createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1501 //
1502 //        IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1503 //        configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1504 //        configurator.setMatchMode(MatchMode.BEGINNING);
1505 //        configurator.setDoTaxa(false);
1506 //        configurator.setDoSynonyms(false);
1507 //        configurator.setDoNamesWithoutTaxa(false);
1508 //        configurator.setDoTaxaByCommonNames(true);
1509 //
1510 //        Pager<IdentifiableEntity> pager;
1511 //
1512 //        long startMillis = System.currentTimeMillis();
1513 //        for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1514 //            pager = taxonService.findTaxaAndNames(configurator);
1515 //            if (logger.isDebugEnabled()) {
1516 //                logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1517 //            }
1518 //        }
1519 //        double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1520 //        logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1521 //    }
1522
1523     @SuppressWarnings("rawtypes")
1524     @Test
1525     @DataSet
1526     public final void benchmarkFindByCommonNameHql() {
1527
1528 //        printDataSet(System.err, new String[] { "TaxonBase" });
1529
1530         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1531
1532         IFindTaxaAndNamesConfigurator configurator = FindTaxaAndNamesConfiguratorImpl.NewInstance();
1533         configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1534         configurator.setMatchMode(MatchMode.BEGINNING);
1535         configurator.setDoTaxa(false);
1536         configurator.setDoSynonyms(false);
1537         configurator.setDoNamesWithoutTaxa(false);
1538         configurator.setDoTaxaByCommonNames(true);
1539
1540         Pager<IdentifiableEntity> pager;
1541
1542         long startMillis = System.currentTimeMillis();
1543         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1544             pager = taxonService.findTaxaAndNames(configurator);
1545             if (logger.isDebugEnabled()) {
1546                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1547             }
1548         }
1549         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1550         logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1551     }
1552
1553     @SuppressWarnings("rawtypes")
1554     @Test
1555     @DataSet
1556     public final void benchmarkFindByCommonNameLucene() throws IOException, LuceneParseException {
1557         TaxonNode subtree = null;
1558         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1559
1560         refreshLuceneIndex();
1561
1562         Pager<SearchResult<TaxonBase>> pager;
1563
1564         long startMillis = System.currentTimeMillis();
1565         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1566             pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, subtree, null, null, false, null, null, null, null);
1567             if (logger.isDebugEnabled()) {
1568                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getEntity().getTitleCache());
1569             }
1570         }
1571         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1572         logger.info("Benchmark result - [find taxon by CommonName via lucene] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1573     }
1574
1575     /**
1576      * uncomment @Test annotation to create the dataset for this test
1577      */
1578     @Override
1579     //    @Test
1580     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="ClearDBDataSet.xml")
1581     public final void createTestDataSet() throws FileNotFoundException {
1582
1583         Classification europeanAbiesClassification = Classification.NewInstance("European Abies");
1584         europeanAbiesClassification.setUuid(CLASSIFICATION_UUID);
1585         classificationService.save(europeanAbiesClassification);
1586
1587         Classification alternativeClassification = Classification.NewInstance("Abies alternative");
1588         alternativeClassification.setUuid(CLASSIFICATION_ALT_UUID);
1589         classificationService.save(alternativeClassification);
1590
1591         Reference sec = ReferenceFactory.newBook();
1592         sec.setTitleCache("Kohlbecker, A., Testcase standart views, 2013", true);
1593         Reference sec_sensu = ReferenceFactory.newBook();
1594         sec_sensu.setTitleCache("Komarov, V. L., Flora SSSR 29", true);
1595         referenceService.save(sec);
1596         referenceService.save(sec_sensu);
1597
1598         IBotanicalName n_abies = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
1599         n_abies.setNameCache("Abies", true);
1600         Taxon t_abies = Taxon.NewInstance(n_abies, sec);
1601         taxonService.save(t_abies);
1602
1603         IBotanicalName n_abies_alba = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1604         n_abies_alba.setNameCache("Abies alba", true);
1605         Taxon t_abies_alba = Taxon.NewInstance(n_abies_alba, sec);
1606         t_abies_alba.setUuid(ABIES_ALBA_UUID);
1607         taxonService.save(t_abies_alba);
1608
1609         IBotanicalName n_abies_subalpina = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1610         n_abies_subalpina.setNameCache("Abies subalpina", true);
1611         Synonym s_abies_subalpina = Synonym.NewInstance(n_abies_subalpina, sec);
1612         taxonService.save(s_abies_subalpina);
1613
1614         IBotanicalName n_abies_balsamea = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1615         n_abies_balsamea.setNameCache("Abies balsamea", true);
1616         Taxon t_abies_balsamea = Taxon.NewInstance(n_abies_balsamea, sec);
1617         t_abies_balsamea.setUuid(ABIES_BALSAMEA_UUID);
1618         t_abies_balsamea.addSynonym(s_abies_subalpina, SynonymType.SYNONYM_OF());
1619         taxonService.save(t_abies_balsamea);
1620
1621         IBotanicalName n_abies_grandis = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1622         n_abies_grandis.setNameCache("Abies grandis", true);
1623         Taxon t_abies_grandis = Taxon.NewInstance(n_abies_grandis, sec);
1624         taxonService.save(t_abies_grandis);
1625
1626         IBotanicalName n_abies_kawakamii = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1627         n_abies_kawakamii.setNameCache("Abies kawakamii", true);
1628         Taxon t_abies_kawakamii = Taxon.NewInstance(n_abies_kawakamii, sec);
1629         t_abies_kawakamii.getTitleCache();
1630         taxonService.save(t_abies_kawakamii);
1631
1632         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1633         Taxon t_abies_kawakamii_sensu_komarov = Taxon.NewInstance(n_abies_kawakamii, sec_sensu);
1634         taxonService.save(t_abies_kawakamii_sensu_komarov);
1635         t_abies_kawakamii_sensu_komarov.addTaxonRelation(t_abies_balsamea, TaxonRelationshipType.MISAPPLIED_NAME_FOR(), null, null);
1636         taxonService.saveOrUpdate(t_abies_kawakamii_sensu_komarov);
1637
1638         IBotanicalName n_abies_lasiocarpa = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1639         n_abies_lasiocarpa.setNameCache("Abies lasiocarpa", true);
1640         Taxon t_abies_lasiocarpa = Taxon.NewInstance(n_abies_lasiocarpa, sec);
1641         taxonService.save(t_abies_lasiocarpa);
1642
1643         // add taxa to classifications
1644         europeanAbiesClassification.addChildTaxon(t_abies_balsamea, null, null);
1645         alternativeClassification.addChildTaxon(t_abies_lasiocarpa, null, null);
1646         classificationService.saveOrUpdate(europeanAbiesClassification);
1647         classificationService.saveOrUpdate(alternativeClassification);
1648
1649         //
1650         // Description
1651         //
1652         TaxonDescription d_abies_alba = TaxonDescription.NewInstance(t_abies_alba);
1653         TaxonDescription d_abies_balsamea = TaxonDescription.NewInstance(t_abies_balsamea);
1654
1655         d_abies_alba.setUuid(DESC_ABIES_ALBA_UUID);
1656         d_abies_balsamea.setUuid(DESC_ABIES_BALSAMEA_UUID);
1657
1658
1659         // CommonTaxonName
1660         d_abies_alba.addElement(CommonTaxonName.NewInstance("Wei"+UTF8.SHARP_S+"tanne", Language.GERMAN()));
1661         d_abies_alba.addElement(CommonTaxonName.NewInstance("silver fir", Language.ENGLISH()));
1662         d_abies_alba.addElement(Distribution
1663                 .NewInstance(
1664                         germany,
1665                         PresenceAbsenceTerm.NATIVE()));
1666         d_abies_alba.addElement(Distribution
1667                 .NewInstance(
1668                         russia,
1669                         PresenceAbsenceTerm.ABSENT()));
1670
1671         // TextData
1672         d_abies_balsamea
1673             .addElement(TextData
1674                     .NewInstance(
1675                             "Die Balsam-Tanne (Abies balsamea) ist eine Pflanzenart aus der Gattung der Tannen (Abies). Sie wÃ¤chst im nordÃ¶stlichen Nordamerika, wo sie sowohl Tief- als auch Bergland besiedelt. Sie gilt als relativ anspruchslos gegenÃ¼ber dem Standort und ist frosthart. In vielen Teilen des natÃ¼rlichen Verbreitungsgebietes stellt sie die Klimaxbaumart dar.",
1676                             Language.GERMAN(), null));
1677         d_abies_balsamea
1678         .addElement(CommonTaxonName
1679                 .NewInstance(
1680                         "Balsam-Tanne",
1681                         Language.GERMAN(), null));
1682
1683         d_abies_balsamea
1684         .addElement(Distribution
1685                 .NewInstance(
1686                         canada,
1687                         PresenceAbsenceTerm.PRESENT()));
1688
1689         d_abies_balsamea
1690         .addElement(Distribution
1691                 .NewInstance(
1692                         germany,
1693                         PresenceAbsenceTerm.NATIVE()));
1694
1695         d_abies_balsamea
1696                 .addElement(TextData
1697                         .NewInstance(
1698                                 TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_LONG,
1699                                 Language.RUSSIAN(), null));
1700         d_abies_balsamea
1701         .addElement(CommonTaxonName
1702                 .NewInstance(
1703                         TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_SHORT,
1704                         Language.RUSSIAN(), null));
1705         descriptionService.saveOrUpdate(d_abies_balsamea);
1706
1707         setComplete();
1708         endTransaction();
1709
1710
1711         writeDbUnitDataSetFile(new String[] {
1712             "TAXONBASE", "TAXONNAME",
1713             "TAXONRELATIONSHIP",
1714             "REFERENCE", "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
1715             "AGENTBASE", "HOMOTYPICALGROUP",
1716             "CLASSIFICATION", "TAXONNODE",
1717             "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING",
1718             "HIBERNATE_SEQUENCES" // IMPORTANT!!!
1719             });
1720
1721     }
1722
1723     /**
1724      *
1725      */
1726     private void refreshLuceneIndex() {
1727
1728 //        commitAndStartNewTransaction(null);
1729         commit();
1730         endTransaction();
1731         indexer.purge(DefaultProgressMonitor.NewInstance());
1732         indexer.reindex(typesToIndex, DefaultProgressMonitor.NewInstance());
1733         startNewTransaction();
1734 //        commitAndStartNewTransaction(null);
1735     }
1736
1737     /**
1738      * @param numberOfNew
1739      *
1740      */
1741     private void createRandomTaxonWithCommonName(int numberOfNew) {
1742
1743         logger.debug(String.format("creating %1$s random taxan with CommonName", numberOfNew));
1744
1745         commitAndStartNewTransaction(null);
1746
1747         Reference sec = ReferenceFactory.newBook();
1748         referenceService.save(sec);
1749
1750         for (int i = numberOfNew; i < numberOfNew; i++) {
1751             RandomStringUtils.randomAlphabetic(10);
1752             String radomName = RandomStringUtils.randomAlphabetic(5) + " " + RandomStringUtils.randomAlphabetic(10);
1753             String radomCommonName = RandomStringUtils.randomAlphabetic(10);
1754
1755             IBotanicalName name = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1756             name.setNameCache(radomName, true);
1757             Taxon taxon = Taxon.NewInstance(name, sec);
1758             taxonService.save(taxon);
1759
1760             TaxonDescription description = TaxonDescription.NewInstance(taxon);
1761             description.addElement(CommonTaxonName.NewInstance(radomCommonName, Language.GERMAN()));
1762             descriptionService.save(description);
1763         }
1764
1765         commitAndStartNewTransaction(null);
1766     }
1767
1768     private <T extends CdmBase> void logFreeTextSearchResults(Pager<SearchResult<T>> pager, Level level, String[] docFields){
1769         if(level == null){
1770             level = Level.DEBUG;
1771         }
1772         if(logger.isEnabledFor(level)){
1773             StringBuilder b = new StringBuilder();
1774             b.append("\n");
1775             int i = 0;
1776             for(SearchResult<?> sr : pager.getRecords()){
1777
1778                 b.append(" ").append(i++).append(" - ");
1779                 b.append("score:").append(sr.getScore()).append(", ");
1780
1781                 if(docFields != null){
1782                     b.append("docs : ");
1783                     for(Document doc : sr.getDocs()) {
1784                         b.append("<");
1785                         for(String f : docFields){
1786                             b.append(f).append(":").append(Arrays.toString(doc.getValues(f)));
1787                         }
1788                         b.append(">");
1789                     }
1790                 }
1791
1792                 CdmBase entity = sr.getEntity();
1793                 if(entity == null){
1794                     b.append("NULL");
1795                 } else {
1796                     b.append(entity.getClass().getSimpleName()).
1797                         append(" [").append(entity.getId()).
1798                         append(" | ").append(entity.getUuid()).append("] : ").
1799                         append(entity.toString());
1800
1801                 }
1802                 b.append("\n");
1803             }
1804             logger.log(level, b);
1805         }
1806     }
1807
1808
1809     /**
1810      * @param pager
1811      * @return
1812      */
1813     private Set<UUID> getTaxonUuidSet(@SuppressWarnings("rawtypes") Pager<SearchResult<TaxonBase>> pager) {
1814         Set<UUID> result = new HashSet<>();
1815         for (@SuppressWarnings("rawtypes") SearchResult<TaxonBase> searchResult : pager.getRecords()){
1816             result.add(searchResult.getEntity().getUuid());
1817         }
1818         return result;
1819     }
1820
1821 }