cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/TaxonServiceSearchTest.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import static org.junit.Assert.assertNotNull;
  13
  14 import java.io.FileNotFoundException;
  15 import java.io.IOException;
  16 import java.util.ArrayList;
  17 import java.util.Arrays;
  18 import java.util.EnumSet;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.RandomStringUtils;
  26 import org.apache.log4j.Level;
  27 import org.apache.log4j.Logger;
  28 import org.apache.lucene.document.Document;
  29 import org.junit.Assert;
  30 import org.junit.Before;
  31 import org.junit.Ignore;
  32 import org.junit.Test;
  33 import org.unitils.dbunit.annotation.DataSet;
  34 import org.unitils.spring.annotation.SpringBeanByType;
  35
  36 import eu.etaxonomy.cdm.api.service.config.FindTaxaAndNamesConfiguratorImpl;
  37 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  38 import eu.etaxonomy.cdm.api.service.pager.Pager;
  39 import eu.etaxonomy.cdm.api.service.search.ICdmMassIndexer;
  40 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  41 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  42 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  43 import eu.etaxonomy.cdm.common.UTF8;
  44 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  45 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  46 import eu.etaxonomy.cdm.model.common.CdmBase;
  47 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  48 import eu.etaxonomy.cdm.model.common.Language;
  49 import eu.etaxonomy.cdm.model.description.CategoricalData;
  50 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  51 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  52 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  53 import eu.etaxonomy.cdm.model.description.Distribution;
  54 import eu.etaxonomy.cdm.model.description.Feature;
  55 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  56 import eu.etaxonomy.cdm.model.description.State;
  57 import eu.etaxonomy.cdm.model.description.StateData;
  58 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  59 import eu.etaxonomy.cdm.model.description.TextData;
  60 import eu.etaxonomy.cdm.model.location.Country;
  61 import eu.etaxonomy.cdm.model.location.NamedArea;
  62 import eu.etaxonomy.cdm.model.name.IBotanicalName;
  63 import eu.etaxonomy.cdm.model.name.Rank;
  64 import eu.etaxonomy.cdm.model.name.TaxonName;
  65 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  66 import eu.etaxonomy.cdm.model.reference.Reference;
  67 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  68 import eu.etaxonomy.cdm.model.taxon.Classification;
  69 import eu.etaxonomy.cdm.model.taxon.Synonym;
  70 import eu.etaxonomy.cdm.model.taxon.SynonymType;
  71 import eu.etaxonomy.cdm.model.taxon.Taxon;
  72 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  73 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  74 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  75 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  76 import eu.etaxonomy.cdm.model.term.DefinedTermBase;
  77 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  78 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  79 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  80 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  81
  82 /**
  83  * @author a.babadshanjan, a.kohlbecker
  84  * @since 04.02.2009
  85  */
  86 public class TaxonServiceSearchTest extends CdmTransactionalIntegrationTest {
  87
  88     private static Logger logger = Logger.getLogger(TaxonServiceSearchTest.class);
  89
  90     private static final int BENCHMARK_ROUNDS = 300;
  91
  92     private static final UUID ABIES_BALSAMEA_UUID = UUID.fromString("f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8");
  93     private static final UUID ABIES_ALBA_UUID = UUID.fromString("7dbd5810-a3e5-44b6-b563-25152b8867f4");
  94     private static final UUID CLASSIFICATION_UUID = UUID.fromString("2a5ceebb-4830-4524-b330-78461bf8cb6b");
  95     private static final UUID CLASSIFICATION_ALT_UUID = UUID.fromString("d7c741e3-ae9e-4a7d-a566-9e3a7a0b51ce");
  96     private static final UUID ABIES_SUBALPINA_UUID = UUID.fromString("9fee273c-c819-4f1f-913a-cd910465df51");
  97     private static final UUID ABIES_LASIOCARPA_UUID = UUID.fromString("9ce1fecf-c1ad-4127-be01-85d5d9f847ce");
  98
  99     private static final UUID ROOTNODE_CLASSIFICATION_5000 = UUID.fromString("a8266e45-091f-432f-87ae-c625e6aa9bbc");
 100
 101     private static final UUID DESC_ABIES_BALSAMEA_UUID = UUID.fromString("900108d8-e6ce-495e-b32e-7aad3099135e");
 102     private static final UUID DESC_ABIES_ALBA_UUID = UUID.fromString("ec8bba03-d993-4c85-8472-18b14942464b");
 103     private static final UUID DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID = UUID.fromString("e9d8c2fd-6409-46d5-9c2e-14a2bbb1b2b1");
 104
 105     private static final int NUM_OF_NEW_RADOM_ENTITIES = 1000;
 106
 107     private boolean includeUnpublished = true;
 108
 109     @SpringBeanByType
 110     private ITaxonService taxonService;
 111     @SpringBeanByType
 112     private ITermService termService;
 113     @SpringBeanByType
 114     private IClassificationService classificationService;
 115     @SpringBeanByType
 116     private IReferenceService referenceService;
 117     @SpringBeanByType
 118     private IDescriptionService descriptionService;
 119     @SpringBeanByType
 120     private IDescriptionElementService descriptionElementService;
 121     @SpringBeanByType
 122     private INameService nameService;
 123     @SpringBeanByType
 124     private ITaxonNodeService nodeService;
 125
 126     @SpringBeanByType
 127     private ICdmMassIndexer indexer;
 128
 129     private Set<Class<? extends CdmBase>> typesToIndex = null;
 130
 131     private NamedArea germany;
 132     private NamedArea france;
 133     private NamedArea russia;
 134     private NamedArea canada;
 135
 136     @Before
 137     public void setUp() throws Exception {
 138         typesToIndex = new HashSet<>();
 139         typesToIndex.add(DescriptionElementBase.class);
 140         typesToIndex.add(TaxonBase.class);
 141         typesToIndex.add(TaxonRelationship.class);
 142
 143         germany =  Country.GERMANY();
 144         france = Country.FRANCEFRENCHREPUBLIC();
 145         russia = Country.RUSSIANFEDERATION();
 146         canada = Country.CANADA();
 147
 148         includeUnpublished = true;
 149     }
 150
 151     @Test
 152     public void testDbUnitUsageTest() throws Exception {
 153         assertNotNull("taxonService should exist", taxonService);
 154         assertNotNull("nameService should exist", nameService);
 155     }
 156
 157     @SuppressWarnings("rawtypes")
 158     @Test
 159     @DataSet
 160     public final void testPurgeAndReindex() throws IOException, LuceneParseException {
 161
 162         refreshLuceneIndex();
 163         TaxonNode subtree = null;
 164
 165         Pager<SearchResult<TaxonBase>> pager = taxonService.findByFullText(null, "Abies", null, subtree,
 166                 includeUnpublished, null, true, null, null, null, null); // --> 8
 167         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 168
 169         indexer.purge(null);
 170         commitAndStartNewTransaction(null);
 171
 172         pager = taxonService.findByFullText(null, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 0
 173         Assert.assertEquals("Expecting no entities since the index has been purged", 0, pager.getCount().intValue());
 174
 175         indexer.reindex(indexer.indexedClasses(), null);
 176         commitAndStartNewTransaction(null);
 177
 178         pager = taxonService.findByFullText(null, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 8
 179         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 180     }
 181
 182     @SuppressWarnings("rawtypes")
 183     @Test
 184     @DataSet
 185     public final void testFindByDescriptionElementFullText_CommonName() throws IOException,
 186             LuceneParseException {
 187
 188         TaxonNode subtree = null;
 189         refreshLuceneIndex();
 190
 191         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne",
 192                 null, subtree, null, null,
 193                 false, null, null, null, null);
 194         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1,
 195                 pager.getCount().intValue());
 196
 197         // the description containing the Nulltanne has no taxon attached,
 198         // taxon.id = null
 199         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Nulltanne", null, subtree, null, null,
 200                 false, null, null, null, null);
 201         Assert.assertEquals("Expecting no entity when searching for 'Nulltanne' ", 0, pager.getCount().intValue());
 202
 203         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, subtree, null,
 204                 Arrays.asList(new Language[] { Language.GERMAN() }), false, null, null, null, null);
 205         Assert.assertEquals("Expecting one entity when searching in German", 1, pager.getCount().intValue());
 206
 207         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, subtree, null,
 208                 Arrays.asList(new Language[] { Language.RUSSIAN() }), false, null, null, null, null);
 209         Assert.assertEquals("Expecting no entity when searching in Russian", 0, pager.getCount().intValue());
 210     }
 211
 212     @SuppressWarnings("rawtypes")
 213     @Test
 214     @DataSet
 215     public final void testFindByDescriptionElementFullText_Distribution() throws IOException, LuceneParseException {
 216
 217         refreshLuceneIndex();
 218         TaxonNode subtree = null;
 219
 220         // by Area
 221         Pager<SearchResult<TaxonBase>>  pager = taxonService.findByDescriptionElementFullText(null, "Canada", null, subtree, null, null, false, null, null, null, null);
 222         Assert.assertEquals("Expecting one entity when searching for arae 'Canada'", 1, pager.getCount().intValue());
 223         // by Status
 224         pager = taxonService.findByDescriptionElementFullText(null, "present", null, subtree, null, null, false, null, null, null, null);
 225         Assert.assertEquals("Expecting one entity when searching for status 'present'", 1, pager.getCount().intValue());
 226     }
 227
 228     @SuppressWarnings("rawtypes")
 229     @Test
 230     @DataSet
 231     public final void testFindByDescriptionElementFullText_wildcard() throws IOException, LuceneParseException {
 232
 233         refreshLuceneIndex();
 234         TaxonNode subtree = null;
 235
 236         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, subtree, null, null, false, null, null, null, null);
 237         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1, pager.getCount().intValue());
 238     }
 239
 240     /**
 241      * Regression test for #3113 (hibernate search: wildcard query can cause BooleanQuery$TooManyClauses: maxClauseCount is set to 1024)
 242      *
 243      * @throws IOException
 244      * @throws LuceneParseException
 245      */
 246     @SuppressWarnings("rawtypes")
 247     @Test
 248     @DataSet
 249     public final void testFindByDescriptionElementFullText_TooManyClauses() throws IOException, LuceneParseException {
 250
 251         TaxonNode subtree = null;
 252
 253         // generate 1024 terms to reproduce the bug
 254         TaxonDescription description = (TaxonDescription) descriptionService.find(DESC_ABIES_ALBA_UUID);
 255         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 256         while(uniqueRandomStrs.size() < 1024){
 257             uniqueRandomStrs.add(RandomStringUtils.random(10, true, false));
 258         }
 259         for(String rndStr: uniqueRandomStrs){
 260             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 261         }
 262         descriptionService.saveOrUpdate(description);
 263         commitAndStartNewTransaction(null);
 264
 265         refreshLuceneIndex();
 266
 267         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, false, null, null, null, null);
 268         Assert.assertEquals("Expecting all 1024 entities grouped into one SearchResult item when searching for Rot*", 1, pager.getCount().intValue());
 269     }
 270
 271     /**
 272      * Regression test for #3116 (fulltext search: always only one page of results)
 273      *
 274      * @throws IOException
 275      * @throws LuceneParseException
 276      */
 277     @SuppressWarnings("rawtypes")
 278     @Test
 279     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 280     public final void testFullText_Paging() throws IOException, LuceneParseException {
 281
 282         TaxonNode subtree = null;
 283         Reference sec = ReferenceFactory.newDatabase();
 284         referenceService.save(sec);
 285
 286         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 287         int numOfItems = 100;
 288         while(uniqueRandomStrs.size() < numOfItems){
 289             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 290         }
 291
 292         for(String rndStr: uniqueRandomStrs){
 293
 294             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(Rank.SERIES()), sec);
 295             taxon.setTitleCache("Tax" + rndStr, true);
 296             taxonService.save(taxon);
 297
 298             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 299             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 300             descriptionService.saveOrUpdate(description);
 301         }
 302
 303         commitAndStartNewTransaction(new String[]{/*"TAXONBASE"*/});
 304         refreshLuceneIndex();
 305
 306         int pageSize = 10;
 307
 308         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, false, pageSize, null, null, null);
 309         Assert.assertEquals("unexpeted number of pages", Integer.valueOf(numOfItems / pageSize), pager.getPagesAvailable());
 310         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, false, pageSize, 9, null, null);
 311         Assert.assertNotNull("last page must have records", pager.getRecords());
 312         Assert.assertNotNull("last item on last page must exist", pager.getRecords().get(0));
 313         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, false, pageSize, 10, null, null);
 314         Assert.assertNotNull("last page + 1 must not have any records", pager.getRecords());
 315     }
 316
 317     /**
 318      * test for max score and sort by score of hit groups
 319      * with all matches per taxon in a single TextData  element
 320      * see {@link #testFullText_ScoreAndOrder_2()} for the complement
 321      * test with matches in multiple TextData per taxon
 322      *
 323      * @throws IOException
 324      * @throws LuceneParseException
 325      */
 326     @SuppressWarnings("rawtypes")
 327     @Test
 328     @DataSet
 329     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 330     public final void testFullText_ScoreAndOrder_1() throws IOException, LuceneParseException {
 331
 332         TaxonNode subtree = null;
 333         int numOfTaxa = 3;
 334
 335         UUID[] taxonUuids = new UUID[numOfTaxa];
 336         StringBuilder text = new StringBuilder();
 337
 338         for(int i = 0; i < numOfTaxa; i++){
 339
 340             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 341             taxon.setTitleCache("Taxon_" + i, true);
 342             taxonUuids[i] = taxon.getUuid();
 343             taxonService.save(taxon);
 344
 345             text.append(" ").append("Rot");
 346             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 347             description.addElement(TextData.NewInstance(text.toString(), Language.DEFAULT(), null));
 348             descriptionService.saveOrUpdate(description);
 349         }
 350
 351         commitAndStartNewTransaction(null);
 352         refreshLuceneIndex();
 353
 354         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, subtree, null, null, false, null, null, null, null);
 355         for(int i = 0; i < numOfTaxa; i++){
 356             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 357         }
 358         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 359     }
 360
 361     /**
 362      * test for max score and sort by score of hit groups
 363      * with all matches per taxon in a multiple TextData elements
 364      * see {@link #testFullText_ScoreAndOrder_1()} for the complement
 365      * test with matches in a single TextData per taxon
 366      *
 367      * @throws IOException
 368      * @throws LuceneParseException
 369      */
 370     @SuppressWarnings("rawtypes")
 371     @Test
 372     @DataSet
 373     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 374     public final void testFullText_ScoreAndOrder_2() throws IOException, LuceneParseException {
 375
 376         TaxonNode subtree = null;
 377         int numOfTaxa = 3;
 378
 379         UUID[] taxonUuids = new UUID[numOfTaxa];
 380
 381         for(int i = 0; i < numOfTaxa; i++){
 382
 383             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 384             taxon.setTitleCache("Taxon_" + i, true);
 385             taxonUuids[i] = taxon.getUuid();
 386             taxonService.save(taxon);
 387
 388             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 389             for(int k = 0; k < i; k++){
 390                 description.addElement(TextData.NewInstance("Rot", Language.DEFAULT(), null));
 391             }
 392             descriptionService.saveOrUpdate(description);
 393         }
 394
 395         commitAndStartNewTransaction(null);
 396         refreshLuceneIndex();
 397
 398         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, subtree, null, null, false, null, null, null, null);
 399         for(int i = 0; i < numOfTaxa; i++){
 400             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 401         }
 402         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 403     }
 404
 405
 406     /**
 407      * @throws IOException
 408      * @throws LuceneParseException
 409      * @throws LuceneMultiSearchException
 410      */
 411     @Test
 412     @DataSet
 413     public final void testFullText_Grouping() throws IOException, LuceneParseException, LuceneMultiSearchException {
 414
 415         TaxonNode subtree = null;
 416         TaxonDescription description = (TaxonDescription) descriptionService.find(DESC_ABIES_ALBA_UUID);
 417         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 418         int numOfItems = 100;
 419         while(uniqueRandomStrs.size() < numOfItems){
 420             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 421         }
 422         for(String rndStr: uniqueRandomStrs){
 423             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 424         }
 425         descriptionService.saveOrUpdate(description);
 426
 427         commitAndStartNewTransaction(new String[]{"DESCRIPTIONELEMENTBASE"});
 428
 429         refreshLuceneIndex();
 430
 431         int pageSize = 10;
 432
 433          boolean highlightFragments = true;
 434
 435         // test with findByDescriptionElementFullText
 436         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, subtree, null, null, highlightFragments, pageSize, null, null, null);
 437         logFreeTextSearchResults(pager, Level.DEBUG, null);
 438         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 439         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 440         Map<String, String[]> highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 441         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 442         int maxDocsPerGroup = 10;
 443         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 444
 445         // test with findByEverythingFullText
 446         pager = taxonService.findByEverythingFullText( "Rot*", null, subtree, includeUnpublished, null, highlightFragments, pageSize, null, null, null);
 447         logFreeTextSearchResults(pager, Level.DEBUG, null);
 448         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 449         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 450         highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 451         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 452         maxDocsPerGroup = 10;
 453         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 454
 455     }
 456
 457     @SuppressWarnings("rawtypes")
 458     @Test
 459     @DataSet
 460     public final void testFindByDescriptionElementFullText_TextData() throws IOException, LuceneParseException {
 461
 462         refreshLuceneIndex();
 463         TaxonNode subtree = null;
 464
 465         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, subtree, null, null, false, null, null, null, null);
 466         logFreeTextSearchResults(pager, Level.DEBUG, null);
 467         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 468         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 469         Assert.assertTrue("Expecting two docs, one for RUSSIAN and one for GERMAN", pager.getRecords().get(0).getDocs().size() == 2);
 470         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 471
 472
 473         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, subtree, null, null, false, null, null, null, null);
 474         Assert.assertEquals("Expecting one entity when searching for any type", 1, pager.getCount().intValue());
 475
 476         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, subtree, Arrays.asList(new Feature[]{Feature.UNKNOWN()}), null, false, null, null, null, null);
 477         Assert.assertEquals("Expecting one entity when searching for any type and for Feature DESCRIPTION", 1, pager.getCount().intValue());
 478
 479         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, subtree, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER()}), null, false, null, null, null, null);
 480         Assert.assertEquals("Expecting no entity when searching for any type and for Feature CHROMOSOME_NUMBER", 0, pager.getCount().intValue());
 481
 482         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, subtree, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER(), Feature.UNKNOWN()}), null, false, null, null, null, null);
 483         Assert.assertEquals("Expecting no entity when searching for any type and for Feature DESCRIPTION or CHROMOSOME_NUMBER", 1, pager.getCount().intValue());
 484
 485         pager = taxonService.findByDescriptionElementFullText(Distribution.class, "Abies", null, subtree, null, null, false, null, null, null, null);
 486         Assert.assertEquals("Expecting no entity when searching for Distribution", 0, pager.getCount().intValue());
 487
 488         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, subtree, null, Arrays.asList(new Language[]{}), false, null, null, null, null);
 489         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 490         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 491
 492         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, subtree, null, Arrays.asList(new Language[]{Language.RUSSIAN()}), false, null, null, null, null);
 493         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 494         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 495
 496         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 497         Assert.assertEquals("Expecting no entity", 0, pager.getCount().intValue());
 498
 499         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 500         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 501         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 502     }
 503
 504     @SuppressWarnings("rawtypes")
 505     @Test
 506     @DataSet
 507     public final void testFindByDescriptionElementFullText_MultipleWords() throws IOException, LuceneParseException {
 508
 509         refreshLuceneIndex();
 510         TaxonNode subtree = null;
 511
 512         // Pflanzenart aus der Gattung der Tannen
 513         long start = System.currentTimeMillis();
 514
 515         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, subtree, null, null, false, null, null, null, null);
 516         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 517
 518         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Wespen", null, subtree, null, null, false, null, null, null, null);
 519         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 520
 521         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, subtree, null, null, false, null, null, null, null);
 522         Assert.assertEquals("AND search : Expecting one entity", 1, pager.getCount().intValue());
 523
 524         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Wespen", null, subtree, null, null, false, null, null, null, null);
 525         Assert.assertEquals("AND search : Expecting no entity", 0, pager.getCount().intValue());
 526
 527         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, subtree, null, null, false, null, null, null, null);
 528         Assert.assertEquals("Phrase search : Expecting one entity", 1, pager.getCount().intValue());
 529
 530         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Wespen\"", null, subtree, null, null, false, null, null, null, null);
 531         Assert.assertEquals("Phrase search : Expecting one entity", 0, pager.getCount().intValue());
 532
 533         logger.info("testFindByDescriptionElementFullText_MultipleWords() duration: " + (System.currentTimeMillis() - start) + "ms");
 534
 535     }
 536
 537
 538     @SuppressWarnings("rawtypes")
 539     @Test
 540     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 541     public final void testFindByDescriptionElementFullText_modify_DescriptionElement() throws IOException, LuceneParseException {
 542
 543         refreshLuceneIndex();
 544         TaxonNode subtree = null;
 545
 546         //
 547         // modify the DescriptionElement
 548         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 549         Assert.assertTrue("Search did not return any results", pager.getRecords().size() > 0);
 550         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 551         Document indexDocument = pager.getRecords().get(0).getDocs().iterator().next();
 552         String[] descriptionElementUuidStr = indexDocument.getValues("uuid");
 553         String[] inDescriptionUuidStr = indexDocument.getValues("inDescription.uuid");
 554         // is only one uuid!
 555         DescriptionElementBase textData = descriptionElementService.find(UUID.fromString(descriptionElementUuidStr[0]));
 556
 557         ((TextData)textData).removeText(Language.GERMAN());
 558         ((TextData)textData).putText(Language.SPANISH_CASTILIAN(), "abeto bals"+UTF8.SMALL_A_ACUTE+"mico");
 559
 560         descriptionElementService.save(textData);
 561         commitAndStartNewTransaction(null);
 562 //        printDataSet(System.out, new String[] {
 563 //                "DESCRIPTIONELEMENTBASE", "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING" }
 564 //        );
 565
 566         //
 567         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 568         Assert.assertEquals("The german 'Balsam-Tanne' TextData should no longer be indexed", 0, pager.getCount().intValue());
 569         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, subtree, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 570         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 571         pager = taxonService.findByDescriptionElementFullText(TextData.class, "bals"+UTF8.SMALL_A_ACUTE+"mico", null, subtree, null, null, false, null, null, null, null);
 572         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 573
 574         //
 575         // modify the DescriptionElement via the Description object
 576         DescriptionBase<?> description = descriptionService.find(UUID.fromString(inDescriptionUuidStr[0]));
 577         Set<DescriptionElementBase> elements = description.getElements();
 578         for( DescriptionElementBase elm : elements){
 579             if(elm.getUuid().toString().equals(descriptionElementUuidStr[0])){
 580                 ((TextData)elm).removeText(Language.SPANISH_CASTILIAN());
 581                 ((TextData)elm).putText(Language.POLISH(), "Jod"+UTF8.POLISH_L+"a balsamiczna");
 582             }
 583         }
 584         descriptionService.saveOrUpdate(description);
 585         commitAndStartNewTransaction(null);
 586         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, subtree, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 587         Assert.assertEquals("The spanish 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico' TextData should no longer be indexed", 0, pager.getCount().intValue());
 588         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsamiczna", null, subtree, null, Arrays.asList(new Language[]{Language.POLISH()}), false, null, null, null, null);
 589         Assert.assertEquals("expecting to find the POLISH 'Jod"+UTF8.POLISH_L+"a balsamiczna'", 1, pager.getCount().intValue());
 590     }
 591
 592     @SuppressWarnings("rawtypes")
 593     @Test
 594     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 595     public final void testFindByDescriptionElementFullText_modify_Taxon() throws IOException, LuceneParseException {
 596
 597         refreshLuceneIndex();
 598         TaxonNode subtree = null;
 599
 600         Taxon t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
 601         TaxonDescription d_abies_balsamea = (TaxonDescription)descriptionService.find(DESC_ABIES_BALSAMEA_UUID);
 602
 603         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne",
 604                 null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 605         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne'", 1, pager.getCount().intValue());
 606
 607         // exchange the Taxon with another one via the Taxon object
 608         // 1.) remove existing description:
 609         t_abies_balsamea.removeDescription(d_abies_balsamea);
 610
 611         taxonService.saveOrUpdate(t_abies_balsamea);
 612         commitAndStartNewTransaction(null);
 613
 614         t_abies_balsamea = (Taxon)taxonService.find(t_abies_balsamea.getUuid());
 615
 616         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne",
 617                 null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 618         Assert.assertEquals("'Balsam-Tanne' should no longer be found", 0, pager.getCount().intValue());
 619
 620         // 2.) create new description and add to taxon:
 621         TaxonDescription d_abies_balsamea_new = TaxonDescription.NewInstance();
 622         d_abies_balsamea_new
 623                 .addElement(TextData
 624                         .NewInstance(
 625                                 "Die Balsamtanne ist mit bis zu 30 m Höhe ein mittelgro"+UTF8.SHARP_S+"er Baum und kann bis zu 200 Jahre alt werden",
 626                                 Language.GERMAN(), null));
 627         t_abies_balsamea.addDescription(d_abies_balsamea_new);
 628         // set authorshipCache to null to avoid validation exception,
 629         // this is maybe not needed in future,  see ticket #3344
 630         IBotanicalName abies_balsamea = CdmBase.deproxy(t_abies_balsamea.getName(), TaxonName.class);
 631         abies_balsamea.setAuthorshipCache(null);
 632         printDataSet(System.err, new String[] {"LANGUAGESTRING_AUD"});
 633         taxonService.saveOrUpdate(t_abies_balsamea);
 634         commitAndStartNewTransaction(null);
 635
 636 //        printDataSet(System.out, new String[] {
 637 //                "DESCRIPTIONBASE"
 638 //        });
 639
 640         pager = taxonService.findByDescriptionElementFullText(TextData.class, "mittelgro"+UTF8.SHARP_S+"er Baum",
 641                 null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 642         Assert.assertEquals("the taxon should be found via the new Description", 1, pager.getCount().intValue());
 643     }
 644
 645     @SuppressWarnings("rawtypes")
 646     @Test
 647     @DataSet
 648     public final void testFindByDescriptionElementFullText_modify_Classification() throws IOException, LuceneParseException {
 649
 650         refreshLuceneIndex();
 651         TaxonNode subtree = null;
 652
 653         // put taxon into other classification, new taxon node
 654         Classification classification = classificationService.find(CLASSIFICATION_UUID);
 655         Classification alternateClassification = classificationService.find(CLASSIFICATION_ALT_UUID);
 656
 657         // TODO: why is the test failing when the childNode is already retrieved here, and not after the following four lines?
 658         //TaxonNode childNode = classification.getChildNodes().iterator().next();
 659
 660         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 661         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne' even if filtering by classification", 1, pager.getCount().intValue());
 662         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 663         Assert.assertEquals("GERMAN 'Balsam-Tanne' should NOT be found in other classification", 0, pager.getCount().intValue());
 664
 665         // check for the right taxon node
 666         TaxonNode childNode = classification.getChildNodes().iterator().next();
 667         Assert.assertEquals("expecting Abies balsamea sec.", childNode.getTaxon().getUuid(), ABIES_BALSAMEA_UUID);
 668         Assert.assertEquals("expecting default classification", childNode.getClassification().getUuid(), CLASSIFICATION_UUID);
 669
 670         // moving the taxon around, the rootnode is only a proxy
 671         alternateClassification.setRootNode(HibernateProxyHelper.deproxy(alternateClassification.getRootNode(), TaxonNode.class));
 672         alternateClassification.addChildNode(childNode, null, null);
 673
 674         classificationService.saveOrUpdate(alternateClassification);
 675         commitAndStartNewTransaction(null);
 676
 677 //        printDataSet(System.out, new String[] {
 678 //            "TAXONBASE", "TAXONNODE", "CLASSIFICATION"
 679 //        });
 680
 681         // reload classification
 682         classification = classificationService.find(CLASSIFICATION_UUID);
 683         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne",
 684                 alternateClassification, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 685         Assert.assertEquals("GERMAN 'Balsam-Tanne' should now be found in other classification", 1, pager.getCount().intValue());
 686
 687         classification.getChildNodes().clear();
 688         classificationService.saveOrUpdate(classification);
 689         commitAndStartNewTransaction(null);
 690
 691         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne",
 692                 classification, subtree, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 693         Assert.assertEquals("Now the GERMAN 'Balsam-Tanne' should NOT be found in original classification", 0, pager.getCount().intValue());
 694
 695     }
 696
 697     @SuppressWarnings("rawtypes")
 698     @Test
 699     @DataSet
 700     public final void testFindByDescriptionElementFullText_CategoricalData() throws IOException, LuceneParseException {
 701
 702         TaxonNode subtree = null;
 703         // add CategoricalData
 704         DescriptionBase d_abies_balsamea = descriptionService.find(DESC_ABIES_BALSAMEA_UUID);
 705         // Categorical data
 706         CategoricalData cdata = CategoricalData.NewInstance();
 707         cdata.setFeature(Feature.DESCRIPTION());
 708         State state = State.NewInstance("green", "green", "gn");
 709
 710         StateData statedata = StateData.NewInstance(state);
 711         statedata.putModifyingText(Language.ENGLISH(), "always, even during winter");
 712         cdata.addStateData(statedata);
 713         d_abies_balsamea.addElement(cdata);
 714
 715         UUID termUUID = termService.save(state).getUuid();
 716         descriptionService.save(d_abies_balsamea);
 717
 718         commitAndStartNewTransaction(null);
 719
 720 //        printDataSet(System.out, new String[] {
 721 //                 "STATEDATA", "STATEDATA_DEFINEDTERMBASE", "STATEDATA_LANGUAGESTRING", "LANGUAGESTRING"});
 722
 723         refreshLuceneIndex();
 724
 725         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CategoricalData.class, "green", null, subtree, null, null, false, null, null, null, null);
 726         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 727         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 728         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 729         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 730
 731
 732         //TODO modify the StateData
 733         TaxonBase taxon = pager.getRecords().get(0).getEntity();
 734
 735         String newName = "Quercus robur";
 736         taxon.setTitleCache(newName + " sec. ", true);
 737
 738         taxonService.saveOrUpdate(taxon);
 739         commitAndStartNewTransaction(null);
 740
 741         taxon = taxonService.find(taxon.getUuid());
 742         Assert.assertEquals(newName + " sec. ", taxon.getTitleCache());
 743         DefinedTermBase term = termService.find(termUUID);
 744
 745         termService.delete(term);
 746
 747     }
 748
 749     @SuppressWarnings("rawtypes")
 750     @Test
 751     @DataSet
 752     public final void testFindByDescriptionElementFullText_Highlighting() throws IOException, LuceneParseException {
 753
 754         refreshLuceneIndex();
 755         TaxonNode subtree = null;
 756
 757         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, subtree, null, null, true, null, null, null, null);
 758         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 759         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 760         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
 761         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 762         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Abies</B>"));
 763
 764         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, subtree, null, null, true, null, null, null, null);
 765         searchResult = pager.getRecords().get(0);
 766         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 767         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 768         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") || fragments[0].contains("<B>Tannen</B>"));
 769
 770         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, subtree, null, null, true, null, null, null, null);
 771         searchResult = pager.getRecords().get(0);
 772         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 773         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 774         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") && fragments[0].contains("<B>Tannen</B>"));
 775
 776         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, subtree, null, null, true, null, null, null, null);
 777         searchResult = pager.getRecords().get(0);
 778         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 779         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 780         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B> <B>aus</B> <B>der</B> <B>Gattung</B> <B>der</B> <B>Tannen</B>"));
 781
 782         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Gatt*", null, subtree, null, null, true, null, null, null, null);
 783         searchResult = pager.getRecords().get(0);
 784         Assert.assertTrue("Wildcard search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 785         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 786         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Gatt"));
 787     }
 788
 789
 790     @Test
 791     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 792     public final void testFindByFullText() throws IOException, LuceneParseException {
 793
 794         refreshLuceneIndex();
 795
 796         classificationService.find(CLASSIFICATION_UUID);
 797         TaxonNode subtree = null;
 798
 799         boolean NO_UNPUBLISHED = false;
 800
 801         Pager<SearchResult<TaxonBase>> pager = taxonService.findByFullText(null, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 7
 802 //        logFreeTextSearchResults(pager, Level.DEBUG, null);
 803         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 804
 805         //subtree
 806         subtree = nodeService.find(ROOTNODE_CLASSIFICATION_5000);
 807         pager = taxonService.findByFullText(null, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 0
 808         Assert.assertEquals("Expecting 2 entities", 2, pager.getCount().intValue());
 809         subtree = null;
 810
 811         pager = taxonService.findByFullText(null, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 812 //        logFreeTextSearchResults(pager, Level.DEBUG, null);
 813         Assert.assertEquals("Expecting 6 entities", 6, pager.getCount().intValue());
 814         Synonym abiesSubalpina = (Synonym)taxonService.find(ABIES_SUBALPINA_UUID);
 815
 816         //accepted published, syn not published
 817         abiesSubalpina.getAcceptedTaxon().setPublish(true);
 818         commitAndStartNewTransaction();
 819         pager = taxonService.findByFullText(null, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 820         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 821
 822         //accepted published, syn published
 823         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 824         abiesSubalpina.setPublish(true);
 825         commitAndStartNewTransaction();
 826         pager = taxonService.findByFullText(null, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 827         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 828
 829         //accepted not published, syn published
 830         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 831         abiesSubalpina.getAcceptedTaxon().setPublish(false);
 832         commitAndStartNewTransaction();
 833         pager = taxonService.findByFullText(null, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 834         Assert.assertEquals("Expecting 6 entities. Synonym and accepted should not be found, though synonym is published",
 835                 6, pager.getCount().intValue());
 836
 837         pager = taxonService.findByFullText(Taxon.class, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 6
 838         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 839
 840         pager = taxonService.findByFullText(Synonym.class, "Abies", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 1
 841         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 842         pager = taxonService.findByFullText(Synonym.class, "Abies", null, subtree, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 1
 843         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 844
 845         pager = taxonService.findByFullText(TaxonBase.class, "sec", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 7
 846         Assert.assertEquals("Expecting 8 entities", 9, pager.getCount().intValue());
 847
 848         pager = taxonService.findByFullText(null, "genus", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 1
 849         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 850
 851         pager = taxonService.findByFullText(Taxon.class, "subalpina", null, subtree, includeUnpublished, null, true, null, null, null, null); // --> 0
 852         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 853
 854
 855         // synonym in classification ???
 856     }
 857
 858     @Test
 859     @DataSet
 860     public final void testPrepareByAreaSearch() throws IOException, LuceneParseException {
 861
 862         List<PresenceAbsenceTerm> statusFilter = new ArrayList<>();
 863         List<NamedArea> areaFilter = new ArrayList<>();
 864         areaFilter.add(germany);
 865         areaFilter.add(canada);
 866         areaFilter.add(russia);
 867
 868         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDistribution(areaFilter, statusFilter, null, null, 20, 0, null, null);
 869         Assert.assertEquals("Expecting 2 entities", Integer.valueOf(2), Integer.valueOf(pager.getRecords().size()));
 870
 871     }
 872
 873     @Test
 874     @DataSet
 875     public final void testFindTaxaAndNamesByFullText_synonymClassificationSubtree() throws IOException, LuceneParseException, LuceneMultiSearchException {
 876
 877         refreshLuceneIndex();
 878         Classification classification = null;
 879         TaxonNode subtree = null;
 880
 881         //
 882         Pager<SearchResult<TaxonBase>> pager;
 883         EnumSet<TaxaAndNamesSearchMode> taxaAndSynonyms = EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished);
 884         pager = taxonService.findTaxaAndNamesByFullText(
 885                 taxaAndSynonyms, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 886         Assert.assertEquals("doTaxa & doSynonyms & unpublished", 8, pager.getCount().intValue());
 887
 888         EnumSet<TaxaAndNamesSearchMode> taxaOnly = EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished);
 889
 890         //classification
 891         classification = classificationService.find(CLASSIFICATION_UUID);
 892         pager = taxonService.findTaxaAndNamesByFullText(
 893                 taxaAndSynonyms, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 894         Assert.assertEquals("doTaxa & doSynonyms & unpublished", 2, pager.getCount().intValue());
 895             //taxa only
 896         pager = taxonService.findTaxaAndNamesByFullText(
 897                 taxaOnly, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 898         Assert.assertEquals("doTaxa & unpublished", 1, pager.getCount().intValue());
 899             //synonyms only
 900         pager = taxonService.findTaxaAndNamesByFullText(
 901                 taxaOnly, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 902         Assert.assertEquals("doSynonyms & unpublished", 1, pager.getCount().intValue());
 903
 904         classification = null;
 905
 906         //subtree
 907        subtree = nodeService.find(ROOTNODE_CLASSIFICATION_5000);
 908        pager = taxonService.findTaxaAndNamesByFullText(
 909                taxaAndSynonyms, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 910        Assert.assertEquals("doTaxa & doSynonyms & unpublished", 2, pager.getCount().intValue());
 911          //taxa only
 912        pager = taxonService.findTaxaAndNamesByFullText(
 913                taxaOnly, "Abies", classification, subtree, null, null, null, true, null, null, null, null);
 914        Assert.assertEquals("doTaxa & unpublished", 1, pager.getCount().intValue());
 915        subtree = null;
 916
 917     }
 918
 919     @Test
 920     @DataSet
 921     public final void testFindTaxaAndNamesByFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
 922
 923         refreshLuceneIndex();
 924         TaxonNode subtree = null;
 925
 926         Classification alternateClassification = classificationService.find(CLASSIFICATION_ALT_UUID);
 927         Synonym abiesSubalpina = (Synonym)taxonService.find(ABIES_SUBALPINA_UUID);
 928
 929         Pager<SearchResult<TaxonBase>> pager;
 930         EnumSet<TaxaAndNamesSearchMode> modes = EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished);
 931         pager = taxonService.findTaxaAndNamesByFullText(
 932                 modes, "Abies", null, subtree, null, null, null, true, null, null, null, null);
 933         Assert.assertEquals("doTaxa & doSynonyms & unpublished", 8, pager.getCount().intValue());
 934 //      logPagerRecords(pager, Level.DEBUG);
 935
 936          //unpublished
 937         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 938                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 939         Assert.assertEquals("doTaxa & doSynonyms, published only", 6, pager.getCount().intValue());
 940
 941         //accepted published, syn not published
 942         abiesSubalpina.getAcceptedTaxon().setPublish(true);
 943         commitAndStartNewTransaction();
 944         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 945                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 946         Assert.assertEquals("doTaxa & doSynonyms, accepted published", 7, pager.getCount().intValue());
 947
 948         //accepted published, syn published
 949         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 950         abiesSubalpina.setPublish(true);
 951         commitAndStartNewTransaction();
 952         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 953                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 954         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 955
 956         //accepted not published, syn published
 957         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 958         abiesSubalpina.getAcceptedTaxon().setPublish(false);
 959         commitAndStartNewTransaction();
 960         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 961                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 962         Assert.assertEquals("Expecting 6 entities. Synonym and accepted should not be found, though synonym is published",
 963                 6, pager.getCount().intValue());
 964
 965         EnumSet<TaxaAndNamesSearchMode> searchMode = EnumSet.allOf(TaxaAndNamesSearchMode.class);
 966         pager = taxonService.findTaxaAndNamesByFullText(
 967                 searchMode, "Abies", null, subtree, null, null, null, true, null, null, null, null);
 968 //        logPagerRecords(pager, Level.DEBUG);
 969         Assert.assertEquals("all search modes", 8, pager.getCount().intValue());
 970         searchMode.remove(TaxaAndNamesSearchMode.includeUnpublished);
 971         pager = taxonService.findTaxaAndNamesByFullText(
 972                 searchMode, "Abies", null, subtree, null, null, null, true, null, null, null, null);
 973         Assert.assertEquals("all search modes except unpublished", 6, pager.getCount().intValue());
 974
 975         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.allOf(TaxaAndNamesSearchMode.class),
 976                 "Abies", alternateClassification, subtree, null, null, null, true, null, null, null, null);
 977 //        logPagerRecords(pager, Level.DEBUG);
 978         Assert.assertEquals("all search modes, filtered by alternateClassification", 1, pager.getCount().intValue());
 979
 980         pager = taxonService.findTaxaAndNamesByFullText(
 981                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 982                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 983         Assert.assertEquals("Expecting 2 entity", 2, pager.getCount().intValue());
 984         Set<UUID> uuids = getTaxonUuidSet(pager);
 985         Assert.assertTrue("The real synonym should be contained", uuids.contains(ABIES_SUBALPINA_UUID));
 986         Assert.assertTrue("The pro parte synonym should be contained",uuids.contains(ABIES_LASIOCARPA_UUID));
 987         //without published
 988         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
 989                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 990         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 991
 992
 993         pager = taxonService.findTaxaAndNamesByFullText(
 994                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
 995                 "Abies", null, subtree, null, null, null, true, null, null, null, null);
 996         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 997
 998         pager = taxonService.findTaxaAndNamesByFullText(
 999                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
1000                 "Tanne", null, subtree, null, null, null, true, null, null, null, null);
1001         Assert.assertEquals("Expecting 1 entity", 1, pager.getRecords().size());
1002         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1003         pager = taxonService.findTaxaAndNamesByFullText(
1004                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
1005                 "Tanne", null, subtree, null, null, null, true, null, null, null, null);
1006         Assert.assertEquals("Expecting 0 entity", 0, pager.getRecords().size());
1007
1008         //misapplied names
1009         pager = taxonService.findTaxaAndNamesByFullText(
1010                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1011                 "kawakamii", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1012         logFreeTextSearchResults(pager, Level.DEBUG, null);
1013         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1014         //unpublish accepted taxon
1015         pager = taxonService.findTaxaAndNamesByFullText(
1016                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1017                 "kawakamii", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1018         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
1019         //published accepted taxon/misapplied name
1020         Taxon abiesBalsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1021         abiesBalsamea.setPublish(true);
1022         commitAndStartNewTransaction();
1023         pager = taxonService.findTaxaAndNamesByFullText(
1024                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1025                 "kawakamii", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1026         Assert.assertEquals("Expecting 1 entities", 1, pager.getCount().intValue());
1027         //unpublished misapplied name
1028         Taxon misapplied = (Taxon)taxonService.find(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1029         misapplied.setPublish(false);
1030         commitAndStartNewTransaction();
1031         pager = taxonService.findTaxaAndNamesByFullText(
1032                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1033                 "kawakamii", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1034         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
1035
1036     }
1037
1038     @Test
1039     @DataSet
1040     public final void testFindTaxaAndNamesByFullText_wildcard() throws IOException, LuceneParseException, LuceneMultiSearchException {
1041
1042         refreshLuceneIndex();
1043         TaxonNode subtree = null;
1044
1045         Pager<SearchResult<TaxonBase>> pager;
1046          pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1047                 "Abi*", null, subtree, null, null, null, true, null, null, null, null);
1048 //      logFreeTextSearchResults(pager, Level.DEBUG, null);
1049         Assert.assertEquals("doTaxa & doSynonyms published only", 6, pager.getCount().intValue());
1050         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1051                 "*bies", null, subtree, null, null, null, true, null, null, null, null);
1052         Assert.assertEquals("doTaxa & doSynonyms, published only", 6, pager.getCount().intValue());
1053         // logFreeTextSearchResults(pager, Level.ERROR, null);
1054         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1055                 "?bies", null, subtree, null, null, null, true, null, null, null, null);
1056         Assert.assertEquals("doTaxa & doSynonyms, published only", 6, pager.getCount().intValue());
1057         // logFreeTextSearchResults(pager, Level.ERROR, null);
1058         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1059                 "*", null, subtree, null, null, null, true, null, null, null, null);
1060         Assert.assertEquals("doTaxa & doSynonyms, published only", 7, pager.getCount().intValue());
1061     }
1062
1063     @Test
1064     @DataSet
1065     // @Ignore // FIXME: fails due  org.apache.lucene.queryparser.classic.ParseException: Cannot parse 'relatedFrom.titleCache:()': Encountered " ")" ") "" at line 1, column 24.
1066     public final void testFindTaxaAndNamesByFullText_empty_querString() throws IOException, LuceneParseException, LuceneMultiSearchException {
1067
1068         refreshLuceneIndex();
1069         TaxonNode subtree = null;
1070
1071         Pager<SearchResult<TaxonBase>> pager;
1072         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
1073                 "", null, subtree, null, null, null, true, null, null, null, null);
1074         Assert.assertEquals("doTaxa, published only", 7, pager.getCount().intValue());
1075
1076         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
1077                 "", null, subtree, null, null, null, true, null, null, null, null);
1078         Assert.assertEquals("doTaxa & doSynonyms, published only", 7, pager.getCount().intValue());
1079
1080         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doMisappliedNames),
1081                 null, null, subtree, null, null, null, true, null, null, null, null);
1082         Assert.assertEquals("doTaxa & doMisappliedNames published only", 7, pager.getCount().intValue());
1083
1084         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.doTaxaByCommonNames),
1085                 null, null, subtree, null, null, null, true, null, null, null, null);
1086         Assert.assertEquals("doTaxa & doMisappliedNames & doTaxaByCommonNames , published only", 7, pager.getCount().intValue());
1087         // logFreeTextSearchResults(pager, Level.ERROR, null);
1088     }
1089
1090     @Test
1091     @DataSet
1092     //test for https://dev.e-taxonomy.eu/redmine/issues/7486
1093     public final void testFindTaxaAndNamesByFullText_synonymsAndMisapplied_7486() throws IOException, LuceneParseException, LuceneMultiSearchException {
1094
1095         refreshLuceneIndex();
1096         TaxonNode subtree = null;
1097
1098         //misapplied names
1099         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1100                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1101                 "Abies", (Classification)null, subtree, null, null, null, true, null, null, null, null);
1102         logFreeTextSearchResults(pager, Level.DEBUG, null);
1103         Assert.assertEquals("Expecting 3 entity", 3, pager.getCount().intValue());
1104         Set<UUID> uuids = getTaxonUuidSet(pager);
1105         Assert.assertTrue("The real synonym should be contained", uuids.contains(ABIES_SUBALPINA_UUID));
1106         Assert.assertTrue("The pro parte synonym should be contained",uuids.contains(ABIES_LASIOCARPA_UUID));
1107         Assert.assertTrue("The misapplied name should be contained",uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1108     }
1109
1110     @Test
1111     @DataSet
1112     public final void testFindTaxaAndNamesByFullText_PhraseQuery() throws IOException, LuceneParseException, LuceneMultiSearchException {
1113
1114         refreshLuceneIndex();
1115         TaxonNode subtree = null;
1116
1117         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1118                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1119                 "\"Abies alba\"", null, subtree, null, null, null, true, null, null, null, null);
1120 //        logPagerRecords(pager, Level.DEBUG);
1121         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 1, pager.getCount().intValue());
1122
1123         pager = taxonService.findTaxaAndNamesByFullText(
1124                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1125                 "\"Abies al*\"", null, subtree, null, null, null, true, null, null, null, null);
1126 //        logPagerRecords(pager, Level.DEBUG);
1127         Assert.assertEquals("doTaxa & doSynonyms with complex phrase query", 1, pager.getCount().intValue());
1128
1129         pager = taxonService.findTaxaAndNamesByFullText(
1130                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1131                 "\"Abies*\"", null, subtree, null, null, null, true, null, null, null, null);
1132 //        logPagerRecords(pager, Level.DEBUG);
1133         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 8, pager.getCount().intValue());
1134
1135         pager = taxonService.findTaxaAndNamesByFullText(
1136                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1137                 "\"Abies*\"", null, subtree, null, null, null, true, null, null, null, null);
1138 //        logPagerRecords(pager, Level.DEBUG);
1139         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 8, pager.getCount().intValue());
1140
1141     }
1142
1143     @Test
1144     @DataSet
1145     public final void testFindTaxaAndNamesByFullText_Sort() throws IOException, LuceneParseException, LuceneMultiSearchException {
1146
1147         refreshLuceneIndex();
1148         TaxonNode subtree = null;
1149
1150         List<OrderHint> orderHints = new ArrayList<>();
1151
1152 //        String[] docFields2log = new String[]{"id"};
1153
1154         // SortById
1155         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
1156         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1157                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
1158                 "Abies", null, subtree, null, null, null, true, null, null, orderHints, null);
1159 //        logSearchResults(pager, Level.DEBUG, docFields2log);
1160         int lastId = -1;
1161         for(SearchResult<TaxonBase> rs : pager.getRecords()){
1162             if(lastId != -1){
1163                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
1164             }
1165             lastId = rs.getEntity().getId();
1166         }
1167
1168         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
1169         pager = taxonService.findTaxaAndNamesByFullText(
1170                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1171                 "Abies", null, subtree, null, null, null, true, null, null, orderHints, null);
1172 //        logSearchResults(pager, Level.DEBUG, docFields2log);
1173
1174         lastId = -1;
1175         for(SearchResult<TaxonBase> rs : pager.getRecords()){
1176             if(lastId != -1){
1177                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
1178             }
1179             lastId = rs.getEntity().getId();
1180         }
1181
1182         // Sortby NOMENCLATURAL_SORT_ORDER TODO make assertions !!!
1183         orderHints.clear();
1184         orderHints.addAll(OrderHint.NOMENCLATURAL_SORT_ORDER.asList());
1185         pager = taxonService.findTaxaAndNamesByFullText(
1186                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1187                 "Abies", null, subtree, null, null, null, true, null, null, orderHints, null);
1188         logFreeTextSearchResults(pager, Level.DEBUG, null);
1189
1190     }
1191
1192     @Test
1193     @DataSet
1194     @Ignore //ignore until #7487 is fixed
1195     public final void testFindTaxaAndNamesByFullText_AreaFilter_7487() throws IOException, LuceneParseException, LuceneMultiSearchException {
1196         refreshLuceneIndex();
1197         TaxonNode subtree = null;
1198         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1199         a_germany_canada_russia.add(germany);
1200         a_germany_canada_russia.add(canada);
1201         a_germany_canada_russia.add(russia);
1202
1203         Set<PresenceAbsenceTerm> present_native = new HashSet<>();
1204         present_native.add(PresenceAbsenceTerm.PRESENT());
1205         present_native.add(PresenceAbsenceTerm.NATIVE());
1206
1207         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1208                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1209                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1210         Assert.assertEquals("Synonyms with matching area filter", 2, pager.getCount().intValue());
1211         Set<UUID> uuids = this.getTaxonUuidSet(pager);
1212         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1213         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1214
1215         //pro parte syn => partial syn
1216         Taxon t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1217         Set<TaxonRelationship> relsTo = t_abies_balsamea.getProParteAndPartialSynonymRelations();
1218         Assert.assertEquals(1, relsTo.size());
1219         TaxonRelationship taxonRelation = relsTo.iterator().next();
1220         taxonRelation.setType(TaxonRelationshipType.PARTIAL_SYNONYM_FOR());
1221         taxonService.saveOrUpdate(t_abies_balsamea);
1222         commitAndStartNewTransaction(null);
1223
1224         //should give same results as above
1225         pager = taxonService.findTaxaAndNamesByFullText(
1226                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1227                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1228 //        Assert.assertEquals("Synonyms with matching area filter", 2, pager.getCount().intValue());
1229 //        uuids = this.getTaxonUuidSet(pager);
1230 //        Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1231 //        Assert.assertTrue("Partial synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1232
1233         ///MISAPPLIED
1234         pager = taxonService.findTaxaAndNamesByFullText(
1235                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished),
1236                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1237         Assert.assertEquals("misappliedNames with matching area & status filter", 3, pager.getCount().intValue());
1238         uuids = this.getTaxonUuidSet(pager);
1239         Assert.assertTrue("Misapplied name should be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1240
1241         t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1242         relsTo = t_abies_balsamea.getMisappliedNameRelations();
1243         Assert.assertEquals(1, relsTo.size());
1244         taxonRelation = relsTo.iterator().next();
1245         Assert.assertEquals(taxonRelation.getFromTaxon().getUuid(), DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1246         taxonRelation.setType(TaxonRelationshipType.PRO_PARTE_MISAPPLIED_NAME_FOR());
1247         taxonService.saveOrUpdate(t_abies_balsamea);
1248         commitAndStartNewTransaction(null);
1249
1250         //strange it works here before fixing #7487 already
1251         pager = taxonService.findTaxaAndNamesByFullText(
1252                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished),
1253                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1254         Assert.assertEquals("misappliedNames with matching area & status filter", 3, pager.getCount().intValue());
1255         uuids = this.getTaxonUuidSet(pager);
1256         Assert.assertTrue("Pro parte misapplied name should be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1257
1258     }
1259
1260
1261     @Test
1262     @DataSet
1263     public final void testFindTaxaAndNamesByFullText_AreaFilter() throws IOException, LuceneParseException, LuceneMultiSearchException {
1264
1265         refreshLuceneIndex();
1266         TaxonNode subtree = null;
1267
1268         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1269         a_germany_canada_russia.add(germany);
1270         a_germany_canada_russia.add(canada);
1271         a_germany_canada_russia.add(russia);
1272
1273         Set<NamedArea> a_russia = new HashSet<>();
1274         a_russia.add(russia);
1275
1276         Set<PresenceAbsenceTerm> present = new HashSet<>();
1277         present.add(PresenceAbsenceTerm.PRESENT());
1278
1279         Set<PresenceAbsenceTerm> present_native = new HashSet<>();
1280         present_native.add(PresenceAbsenceTerm.PRESENT());
1281         present_native.add(PresenceAbsenceTerm.NATIVE());
1282
1283         Set<PresenceAbsenceTerm> absent = new HashSet<>();
1284         absent.add(PresenceAbsenceTerm.ABSENT());
1285
1286         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1287                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished),
1288                 "Abies", null, subtree, a_germany_canada_russia, null, null, true, null, null, null, null);
1289         logFreeTextSearchResults(pager, Level.DEBUG, null);
1290         Assert.assertEquals("Synonyms with matching area filter", 2, pager.getCount().intValue());
1291         Set<UUID> uuids = this.getTaxonUuidSet(pager);
1292         Assert.assertTrue("Abies alba (accepted with distribution) should be in", uuids.contains(ABIES_ALBA_UUID));
1293         Assert.assertTrue("Abies balsamea (accepted with distribution) should be in", uuids.contains(ABIES_BALSAMEA_UUID));
1294
1295         pager = taxonService.findTaxaAndNamesByFullText(
1296                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1297                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1298         Assert.assertEquals("Synonyms with matching area filter", 2, pager.getCount().intValue());
1299         uuids = this.getTaxonUuidSet(pager);
1300         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1301         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1302
1303         pager = taxonService.findTaxaAndNamesByFullText(
1304                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1305                 "Abies", null, subtree, a_germany_canada_russia, null, null, true, null, null, null, null);
1306         logFreeTextSearchResults(pager, Level.DEBUG, null);
1307         Assert.assertEquals("taxa and synonyms with matching area filter", 4, pager.getCount().intValue());
1308         uuids = this.getTaxonUuidSet(pager);
1309         Assert.assertTrue("Accepted taxon with area should be in", uuids.contains(ABIES_ALBA_UUID));
1310         Assert.assertTrue("Accepted taxon with area should be in", uuids.contains(ABIES_BALSAMEA_UUID));
1311         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1312         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1313         Assert.assertFalse("Misapplied name should NOT be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1314
1315         pager = taxonService.findTaxaAndNamesByFullText(
1316                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1317                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1318         Assert.assertEquals("taxa and synonyms with matching area & status filter 4", 4, pager.getCount().intValue());
1319         uuids = this.getTaxonUuidSet(pager);
1320         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1321         Assert.assertTrue("Accepted taxon with area should be in", uuids.contains(ABIES_ALBA_UUID));
1322         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1323         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1324
1325         pager = taxonService.findTaxaAndNamesByFullText(
1326                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1327                 "Abies", null, subtree, a_germany_canada_russia, present, null, true, null, null, null, null);
1328         Assert.assertEquals("taxa and synonyms with matching area & status filter 3", 3, pager.getCount().intValue());
1329         uuids = this.getTaxonUuidSet(pager);
1330         Assert.assertTrue("Abies balsamea (accepted taxon) should be in", uuids.contains(ABIES_BALSAMEA_UUID));
1331         Assert.assertTrue("Synonym of balsamea should be in", uuids.contains(ABIES_SUBALPINA_UUID));
1332         Assert.assertTrue("Pro parte synonym of balsamea should be in", uuids.contains(ABIES_LASIOCARPA_UUID));
1333
1334         pager = taxonService.findTaxaAndNamesByFullText(
1335                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1336                 "Abies", null, subtree, a_russia, present, null, true, null, null, null, null);
1337         Assert.assertEquals("taxa and synonyms with non matching area & status filter", 0, pager.getCount().intValue());
1338
1339         pager = taxonService.findTaxaAndNamesByFullText(
1340                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
1341                 "Tanne", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1342         Assert.assertEquals("ByCommonNames with area filter", 1, pager.getCount().intValue());
1343         uuids = this.getTaxonUuidSet(pager);
1344         Assert.assertTrue("Abies balsamea should be in", uuids.contains(ABIES_BALSAMEA_UUID));
1345
1346         // abies_kawakamii_sensu_komarov as misapplied name for t_abies_balsamea
1347         pager = taxonService.findTaxaAndNamesByFullText(
1348                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1349                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1350         Assert.assertEquals("misappliedNames with matching area & status filter", 1, pager.getCount().intValue());
1351         uuids = this.getTaxonUuidSet(pager);
1352         Assert.assertTrue("Misapplied name should  be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1353
1354
1355         // 1. remove existing taxon relation
1356         Taxon t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1357         Set<TaxonRelationship> relsTo = t_abies_balsamea.getMisappliedNameRelations();
1358         Assert.assertEquals(1, relsTo.size());
1359         TaxonRelationship taxonRelation = relsTo.iterator().next();
1360         t_abies_balsamea.removeTaxonRelation(taxonRelation);
1361         taxonService.saveOrUpdate(t_abies_balsamea);
1362         commitAndStartNewTransaction(null);
1363
1364         pager = taxonService.findTaxaAndNamesByFullText(
1365                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1366                 "Abies", null, subtree, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1367         Assert.assertEquals("misappliedNames with matching area & status filter, should match nothing now", 0, pager.getCount().intValue());
1368
1369         // 2. now add abies_kawakamii_sensu_komarov as misapplied name for t_abies_alba and search for misapplications in Russia: ABSENT
1370         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1371         Taxon t_abies_alba = (Taxon)taxonService.find(ABIES_ALBA_UUID);
1372         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1373         taxonService.update(t_abies_kawakamii_sensu_komarov);
1374         commitAndStartNewTransaction(null);
1375
1376         pager = taxonService.findTaxaAndNamesByFullText(
1377                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1378                 "Abies", null, subtree, a_germany_canada_russia, absent, null, true, null, null, null, null);
1379         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1380         uuids = this.getTaxonUuidSet(pager);
1381         Assert.assertTrue("Misapplied name should  be in", uuids.contains(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1382
1383     }
1384
1385     @Test
1386     @DataSet
1387     //https://dev.e-taxonomy.eu/redmine/issues/5477
1388     public final void testFindTaxaAndNamesByFullText_AreaFilter_issue5477() throws IOException, LuceneParseException, LuceneMultiSearchException {
1389
1390         TaxonNode subtree = null;
1391         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1392         a_germany_canada_russia.add(germany);
1393         a_germany_canada_russia.add(canada);
1394         a_germany_canada_russia.add(russia);
1395
1396
1397         Set<PresenceAbsenceTerm> absent = new HashSet<>();
1398         absent.add(PresenceAbsenceTerm.ABSENT());
1399
1400         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(DESC_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1401         Taxon t_abies_alba = (Taxon)taxonService.find(ABIES_ALBA_UUID);
1402         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1403
1404         /* Since the upgrade from hibernate search 4 to 5.5
1405          * triggering an update of t_abies_alba is no longer sufficient to also update the
1406          * document of t_abies_kawakamii_sensu_komarov in the lucene index.
1407          * the last test in testFindTaxaAndNamesByFullText_AreaFilter() failed in this case.
1408          * This situation is reproduced here:
1409          */
1410         taxonService.update(t_abies_alba);
1411
1412         commitAndStartNewTransaction(null);
1413
1414         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1415                   EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1416                   "Abies", null, subtree, a_germany_canada_russia, absent, null, true, null, null, null, null);
1417         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1418     }
1419
1420
1421     /**
1422      * Regression test for #3119: fulltext search: Entity always null whatever search
1423      *
1424      * @throws IOException
1425      * @throws LuceneParseException
1426      * @throws LuceneMultiSearchException
1427      */
1428     @Test
1429     @DataSet
1430     public final void testFindByEverythingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1431
1432         refreshLuceneIndex();
1433         TaxonNode subtree = null;
1434         EnumSet<TaxaAndNamesSearchMode> mode = TaxaAndNamesSearchMode.taxaAndSynonymsWithUnpublished();
1435         // via Taxon
1436         Pager<SearchResult<TaxonBase>>pager = taxonService.findByEverythingFullText("Abies", null, subtree, includeUnpublished, null, true, null, null, null, null);
1437 //        Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(mode,
1438 //                "Abies", null, null, null, null, true, null, null, null, null);
1439         logFreeTextSearchResults(pager, Level.DEBUG, null);
1440         Assert.assertTrue("Expecting at least 7 entities for 'Abies'", pager.getCount() > 7);
1441         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1442 //        Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1443
1444         // via DescriptionElement
1445         pager = taxonService.findByEverythingFullText("present", null, subtree, includeUnpublished, null, true, null, null, null, null);
1446         //this is not covered by findTaxaAndNamesByFullText
1447 //        pager = taxonService.findTaxaAndNamesByFullText(mode,
1448 //                "present", null, null, null, null, true, null, null, null, null);
1449         Assert.assertEquals("Expecting one entity when searching for area 'present'", 1, pager.getCount().intValue());
1450         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1451         Assert.assertEquals("Expecting Taxon entity", Taxon.class, CdmBase.deproxy(pager.getRecords().get(0).getEntity()).getClass());
1452         Assert.assertEquals("Expecting Taxon ", ABIES_BALSAMEA_UUID, pager.getRecords().get(0).getEntity().getUuid());
1453
1454     }
1455
1456
1457     @Test
1458     @DataSet
1459     public final void findByEveryThingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1460
1461         refreshLuceneIndex();
1462         TaxonNode subtree = null;
1463
1464         Classification classification = null;
1465         EnumSet<TaxaAndNamesSearchMode> mode = TaxaAndNamesSearchMode.taxaAndSynonymsWithUnpublished();
1466
1467         Pager<SearchResult<TaxonBase>> pager = taxonService.findByEverythingFullText("genus", null, subtree, includeUnpublished, null, false, null, null, null, null); // --> 1
1468 //        Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(mode,
1469 //                "genus", classification, null, null, null, false, null, null, null, null);
1470         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1471
1472         //FIXME FAILS: abies balamea is returned twice, see also testFullText_Grouping()
1473         pager = taxonService.findByEverythingFullText("Balsam", null, subtree, includeUnpublished, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1474         logFreeTextSearchResults(pager, Level.DEBUG, null);
1475 //        pager = taxonService.findTaxaAndNamesByFullText(EnumSet.allOf(TaxaAndNamesSearchMode.class),
1476 //                "Balsam", classification, null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1477         Assert.assertEquals("expecting to find the Abies balsamea via the GERMAN DescriptionElements", 1, pager.getCount().intValue());
1478
1479         //TODO fieldHighlight does not yet work
1480         pager = taxonService.findByEverythingFullText("Abies", null, subtree, includeUnpublished, null, true, null, null, null, null);
1481 //        pager = taxonService.findTaxaAndNamesByFullText(mode,
1482 //                "Abies", classification, null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1483         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
1484         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
1485         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
1486         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
1487         Assert.assertTrue("first fragments should contains serch term", fragments[0].toLowerCase().contains("<b>abies</b>"));
1488     }
1489
1490 //    @SuppressWarnings("rawtypes")
1491 //    @Test
1492 //    @DataSet
1493 //    public final void benchmarkFindTaxaAndNamesHql() throws IOException, LuceneParseException {
1494 //
1495 //        createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1496 //
1497 //        IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1498 //        configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1499 //        configurator.setMatchMode(MatchMode.BEGINNING);
1500 //        configurator.setDoTaxa(false);
1501 //        configurator.setDoSynonyms(false);
1502 //        configurator.setDoNamesWithoutTaxa(false);
1503 //        configurator.setDoTaxaByCommonNames(true);
1504 //
1505 //        Pager<IdentifiableEntity> pager;
1506 //
1507 //        long startMillis = System.currentTimeMillis();
1508 //        for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1509 //            pager = taxonService.findTaxaAndNames(configurator);
1510 //            if (logger.isDebugEnabled()) {
1511 //                logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1512 //            }
1513 //        }
1514 //        double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1515 //        logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1516 //    }
1517
1518     @SuppressWarnings("rawtypes")
1519     @Test
1520     @DataSet
1521     public final void benchmarkFindByCommonNameHql() {
1522
1523 //        printDataSet(System.err, new String[] { "TaxonBase" });
1524
1525         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1526
1527         IFindTaxaAndNamesConfigurator configurator = FindTaxaAndNamesConfiguratorImpl.NewInstance();
1528         configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1529         configurator.setMatchMode(MatchMode.BEGINNING);
1530         configurator.setDoTaxa(false);
1531         configurator.setDoSynonyms(false);
1532         configurator.setDoNamesWithoutTaxa(false);
1533         configurator.setDoTaxaByCommonNames(true);
1534
1535         Pager<IdentifiableEntity> pager;
1536
1537         long startMillis = System.currentTimeMillis();
1538         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1539             pager = taxonService.findTaxaAndNames(configurator);
1540             if (logger.isDebugEnabled()) {
1541                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1542             }
1543         }
1544         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1545         logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1546     }
1547
1548     @SuppressWarnings("rawtypes")
1549     @Test
1550     @DataSet
1551     public final void benchmarkFindByCommonNameLucene() throws IOException, LuceneParseException {
1552         TaxonNode subtree = null;
1553         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1554
1555         refreshLuceneIndex();
1556
1557         Pager<SearchResult<TaxonBase>> pager;
1558
1559         long startMillis = System.currentTimeMillis();
1560         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1561             pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, subtree, null, null, false, null, null, null, null);
1562             if (logger.isDebugEnabled()) {
1563                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getEntity().getTitleCache());
1564             }
1565         }
1566         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1567         logger.info("Benchmark result - [find taxon by CommonName via lucene] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1568     }
1569
1570     /**
1571      *
1572      */
1573     private void refreshLuceneIndex() {
1574
1575 //        commitAndStartNewTransaction(null);
1576         commit();
1577         endTransaction();
1578         indexer.purge(DefaultProgressMonitor.NewInstance());
1579         indexer.reindex(typesToIndex, DefaultProgressMonitor.NewInstance());
1580         startNewTransaction();
1581 //        commitAndStartNewTransaction(null);
1582     }
1583
1584     /**
1585      * @param numberOfNew
1586      *
1587      */
1588     private void createRandomTaxonWithCommonName(int numberOfNew) {
1589
1590         logger.debug(String.format("creating %1$s random taxan with CommonName", numberOfNew));
1591
1592         commitAndStartNewTransaction(null);
1593
1594         Reference sec = ReferenceFactory.newBook();
1595         referenceService.save(sec);
1596
1597         for (int i = numberOfNew; i < numberOfNew; i++) {
1598             RandomStringUtils.randomAlphabetic(10);
1599             String radomName = RandomStringUtils.randomAlphabetic(5) + " " + RandomStringUtils.randomAlphabetic(10);
1600             String radomCommonName = RandomStringUtils.randomAlphabetic(10);
1601
1602             IBotanicalName name = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1603             name.setNameCache(radomName, true);
1604             Taxon taxon = Taxon.NewInstance(name, sec);
1605             taxonService.save(taxon);
1606
1607             TaxonDescription description = TaxonDescription.NewInstance(taxon);
1608             description.addElement(CommonTaxonName.NewInstance(radomCommonName, Language.GERMAN()));
1609             descriptionService.save(description);
1610         }
1611
1612         commitAndStartNewTransaction(null);
1613     }
1614
1615     private <T extends CdmBase> void logFreeTextSearchResults(Pager<SearchResult<T>> pager, Level level, String[] docFields){
1616         if(level == null){
1617             level = Level.DEBUG;
1618         }
1619         if(logger.isEnabledFor(level)){
1620             StringBuilder b = new StringBuilder();
1621             b.append("\n");
1622             int i = 0;
1623             for(SearchResult<?> sr : pager.getRecords()){
1624
1625                 b.append(" ").append(i++).append(" - ");
1626                 b.append("score:").append(sr.getScore()).append(", ");
1627
1628                 if(docFields != null){
1629                     b.append("docs : ");
1630                     for(Document doc : sr.getDocs()) {
1631                         b.append("<");
1632                         for(String f : docFields){
1633                             b.append(f).append(":").append(Arrays.toString(doc.getValues(f)));
1634                         }
1635                         b.append(">");
1636                     }
1637                 }
1638
1639                 CdmBase entity = sr.getEntity();
1640                 if(entity == null){
1641                     b.append("NULL");
1642                 } else {
1643                     b.append(entity.getClass().getSimpleName()).
1644                         append(" [").append(entity.getId()).
1645                         append(" | ").append(entity.getUuid()).append("] : ").
1646                         append(entity.toString());
1647
1648                 }
1649                 b.append("\n");
1650             }
1651             logger.log(level, b);
1652         }
1653     }
1654
1655     private Set<UUID> getTaxonUuidSet(@SuppressWarnings("rawtypes") Pager<SearchResult<TaxonBase>> pager) {
1656         Set<UUID> result = new HashSet<>();
1657         for (@SuppressWarnings("rawtypes") SearchResult<TaxonBase> searchResult : pager.getRecords()){
1658             result.add(searchResult.getEntity().getUuid());
1659         }
1660         return result;
1661     }
1662
1663
1664     /**
1665      * uncomment @Test annotation to create the dataset for this test
1666      */
1667     @Override
1668 //    @Test
1669     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="../../database/ClearDBDataSet.xml")
1670     public final void createTestDataSet() throws FileNotFoundException {
1671
1672         Classification europeanAbiesClassification = Classification.NewInstance("European Abies");
1673         europeanAbiesClassification.setUuid(CLASSIFICATION_UUID);
1674         classificationService.save(europeanAbiesClassification);
1675
1676         Classification alternativeClassification = Classification.NewInstance("Abies alternative");
1677         alternativeClassification.setUuid(CLASSIFICATION_ALT_UUID);
1678         classificationService.save(alternativeClassification);
1679
1680         Reference sec = ReferenceFactory.newBook();
1681         sec.setTitleCache("Kohlbecker, A., Testcase standart views, 2013", true);
1682         Reference sec_sensu = ReferenceFactory.newBook();
1683         sec_sensu.setTitleCache("Komarov, V. L., Flora SSSR 29", true);
1684         referenceService.save(sec);
1685         referenceService.save(sec_sensu);
1686
1687         IBotanicalName n_abies = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
1688         n_abies.setNameCache("Abies", true);
1689         Taxon t_abies = Taxon.NewInstance(n_abies, sec);
1690         taxonService.save(t_abies);
1691
1692         IBotanicalName n_abies_alba = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1693         n_abies_alba.setNameCache("Abies alba", true);
1694         Taxon t_abies_alba = Taxon.NewInstance(n_abies_alba, sec);
1695         t_abies_alba.setUuid(ABIES_ALBA_UUID);
1696         taxonService.save(t_abies_alba);
1697
1698         IBotanicalName n_abies_subalpina = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1699         n_abies_subalpina.setNameCache("Abies subalpina", true);
1700         Synonym s_abies_subalpina = Synonym.NewInstance(n_abies_subalpina, sec);
1701         taxonService.save(s_abies_subalpina);
1702
1703         IBotanicalName n_abies_balsamea = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1704         n_abies_balsamea.setNameCache("Abies balsamea", true);
1705         Taxon t_abies_balsamea = Taxon.NewInstance(n_abies_balsamea, sec);
1706         t_abies_balsamea.setUuid(ABIES_BALSAMEA_UUID);
1707         t_abies_balsamea.addSynonym(s_abies_subalpina, SynonymType.SYNONYM_OF());
1708         taxonService.save(t_abies_balsamea);
1709
1710         IBotanicalName n_abies_grandis = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1711         n_abies_grandis.setNameCache("Abies grandis", true);
1712         Taxon t_abies_grandis = Taxon.NewInstance(n_abies_grandis, sec);
1713         taxonService.save(t_abies_grandis);
1714
1715         IBotanicalName n_abies_kawakamii = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1716         n_abies_kawakamii.setNameCache("Abies kawakamii", true);
1717         Taxon t_abies_kawakamii = Taxon.NewInstance(n_abies_kawakamii, sec);
1718         t_abies_kawakamii.getTitleCache();
1719         taxonService.save(t_abies_kawakamii);
1720
1721         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1722         Taxon t_abies_kawakamii_sensu_komarov = Taxon.NewInstance(n_abies_kawakamii, sec_sensu);
1723         taxonService.save(t_abies_kawakamii_sensu_komarov);
1724         t_abies_kawakamii_sensu_komarov.addTaxonRelation(t_abies_balsamea, TaxonRelationshipType.MISAPPLIED_NAME_FOR(), null, null);
1725         taxonService.saveOrUpdate(t_abies_kawakamii_sensu_komarov);
1726
1727         IBotanicalName n_abies_lasiocarpa = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1728         n_abies_lasiocarpa.setNameCache("Abies lasiocarpa", true);
1729         Taxon t_abies_lasiocarpa = Taxon.NewInstance(n_abies_lasiocarpa, sec);
1730         taxonService.save(t_abies_lasiocarpa);
1731
1732         // add taxa to classifications
1733         europeanAbiesClassification.addChildTaxon(t_abies_balsamea, null, null);
1734         alternativeClassification.addChildTaxon(t_abies_lasiocarpa, null, null);
1735         classificationService.saveOrUpdate(europeanAbiesClassification);
1736         classificationService.saveOrUpdate(alternativeClassification);
1737
1738         //
1739         // Description
1740         //
1741         TaxonDescription d_abies_alba = TaxonDescription.NewInstance(t_abies_alba);
1742         TaxonDescription d_abies_balsamea = TaxonDescription.NewInstance(t_abies_balsamea);
1743
1744         d_abies_alba.setUuid(DESC_ABIES_ALBA_UUID);
1745         d_abies_balsamea.setUuid(DESC_ABIES_BALSAMEA_UUID);
1746
1747
1748         // CommonTaxonName
1749         d_abies_alba.addElement(CommonTaxonName.NewInstance("Wei"+UTF8.SHARP_S+"tanne", Language.GERMAN()));
1750         d_abies_alba.addElement(CommonTaxonName.NewInstance("silver fir", Language.ENGLISH()));
1751         d_abies_alba.addElement(Distribution
1752                 .NewInstance(
1753                         germany,
1754                         PresenceAbsenceTerm.NATIVE()));
1755         d_abies_alba.addElement(Distribution
1756                 .NewInstance(
1757                         russia,
1758                         PresenceAbsenceTerm.ABSENT()));
1759
1760         // TextData
1761         d_abies_balsamea
1762             .addElement(TextData
1763                     .NewInstance(
1764                             "Die Balsam-Tanne (Abies balsamea) ist eine Pflanzenart aus der Gattung der Tannen (Abies). Sie wÃ¤chst im nordÃ¶stlichen Nordamerika, wo sie sowohl Tief- als auch Bergland besiedelt. Sie gilt als relativ anspruchslos gegenÃ¼ber dem Standort und ist frosthart. In vielen Teilen des natÃ¼rlichen Verbreitungsgebietes stellt sie die Klimaxbaumart dar.",
1765                             Language.GERMAN(), null));
1766         d_abies_balsamea
1767         .addElement(CommonTaxonName
1768                 .NewInstance(
1769                         "Balsam-Tanne",
1770                         Language.GERMAN(), null));
1771
1772         d_abies_balsamea
1773         .addElement(Distribution
1774                 .NewInstance(
1775                         canada,
1776                         PresenceAbsenceTerm.PRESENT()));
1777
1778         d_abies_balsamea
1779         .addElement(Distribution
1780                 .NewInstance(
1781                         germany,
1782                         PresenceAbsenceTerm.NATIVE()));
1783
1784         d_abies_balsamea
1785                 .addElement(TextData
1786                         .NewInstance(
1787                                 TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_LONG,
1788                                 Language.RUSSIAN(), null));
1789         d_abies_balsamea
1790         .addElement(CommonTaxonName
1791                 .NewInstance(
1792                         TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_SHORT,
1793                         Language.RUSSIAN(), null));
1794         descriptionService.saveOrUpdate(d_abies_balsamea);
1795
1796         setComplete();
1797         endTransaction();
1798
1799
1800         writeDbUnitDataSetFile(new String[] {
1801             "TAXONBASE", "TAXONNAME",
1802             "TAXONRELATIONSHIP",
1803             "REFERENCE", "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
1804             "AGENTBASE", "HOMOTYPICALGROUP",
1805             "CLASSIFICATION", "TAXONNODE",
1806             "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING",
1807             "HIBERNATE_SEQUENCES" // IMPORTANT!!!
1808             });
1809
1810     }
1811
1812 }