cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/TaxonServiceSearchTest.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import static org.junit.Assert.assertNotNull;
  13
  14 import java.io.FileNotFoundException;
  15 import java.io.IOException;
  16 import java.util.ArrayList;
  17 import java.util.Arrays;
  18 import java.util.EnumSet;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.RandomStringUtils;
  26 import org.apache.log4j.Level;
  27 import org.apache.log4j.Logger;
  28 import org.apache.lucene.document.Document;
  29 import org.junit.Assert;
  30 import org.junit.Before;
  31 import org.junit.Ignore;
  32 import org.junit.Test;
  33 import org.unitils.dbunit.annotation.DataSet;
  34 import org.unitils.spring.annotation.SpringBeanByType;
  35
  36 import eu.etaxonomy.cdm.api.service.config.FindTaxaAndNamesConfiguratorImpl;
  37 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  38 import eu.etaxonomy.cdm.api.service.pager.Pager;
  39 import eu.etaxonomy.cdm.api.service.search.ICdmMassIndexer;
  40 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  41 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  42 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  43 import eu.etaxonomy.cdm.common.UTF8;
  44 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  45 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  46 import eu.etaxonomy.cdm.model.common.CdmBase;
  47 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  48 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  49 import eu.etaxonomy.cdm.model.common.Language;
  50 import eu.etaxonomy.cdm.model.description.CategoricalData;
  51 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  52 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  53 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  54 import eu.etaxonomy.cdm.model.description.Distribution;
  55 import eu.etaxonomy.cdm.model.description.Feature;
  56 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  57 import eu.etaxonomy.cdm.model.description.State;
  58 import eu.etaxonomy.cdm.model.description.StateData;
  59 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  60 import eu.etaxonomy.cdm.model.description.TextData;
  61 import eu.etaxonomy.cdm.model.location.Country;
  62 import eu.etaxonomy.cdm.model.location.NamedArea;
  63 import eu.etaxonomy.cdm.model.name.IBotanicalName;
  64 import eu.etaxonomy.cdm.model.name.Rank;
  65 import eu.etaxonomy.cdm.model.name.TaxonName;
  66 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  67 import eu.etaxonomy.cdm.model.reference.Reference;
  68 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  69 import eu.etaxonomy.cdm.model.taxon.Classification;
  70 import eu.etaxonomy.cdm.model.taxon.Synonym;
  71 import eu.etaxonomy.cdm.model.taxon.SynonymType;
  72 import eu.etaxonomy.cdm.model.taxon.Taxon;
  73 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  74 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  75 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  76 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  77 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  78 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  79 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  80 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  81
  82 /**
  83  * @author a.babadshanjan, a.kohlbecker
  84  * @since 04.02.2009
  85  */
  86 public class TaxonServiceSearchTest extends CdmTransactionalIntegrationTest {
  87
  88     private static Logger logger = Logger.getLogger(TaxonServiceSearchTest.class);
  89
  90     private static final String ABIES_BALSAMEA_UUID = "f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8";
  91     private static final String ABIES_ALBA_UUID = "7dbd5810-a3e5-44b6-b563-25152b8867f4";
  92     private static final String CLASSIFICATION_UUID = "2a5ceebb-4830-4524-b330-78461bf8cb6b";
  93     private static final String CLASSIFICATION_ALT_UUID = "d7c741e3-ae9e-4a7d-a566-9e3a7a0b51ce";
  94     private static final String D_ABIES_BALSAMEA_UUID = "900108d8-e6ce-495e-b32e-7aad3099135e";
  95     private static final String D_ABIES_ALBA_UUID = "ec8bba03-d993-4c85-8472-18b14942464b";
  96     private static final String D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID = "e9d8c2fd-6409-46d5-9c2e-14a2bbb1b2b1";
  97     private static final int NUM_OF_NEW_RADOM_ENTITIES = 1000;
  98     private boolean includeUnpublished = true;
  99
 100
 101
 102     @SpringBeanByType
 103     private ITaxonService taxonService;
 104     @SpringBeanByType
 105     private ITermService termService;
 106     @SpringBeanByType
 107     private IClassificationService classificationService;
 108     @SpringBeanByType
 109     private IReferenceService referenceService;
 110     @SpringBeanByType
 111     private IDescriptionService descriptionService;
 112     @SpringBeanByType
 113     private INameService nameService;
 114     @SpringBeanByType
 115     private ICdmMassIndexer indexer;
 116
 117     @SpringBeanByType
 118     private ITaxonNodeService nodeService;
 119
 120     private static final int BENCHMARK_ROUNDS = 300;
 121
 122     private Set<Class<? extends CdmBase>> typesToIndex = null;
 123
 124     private NamedArea germany;
 125     private NamedArea france ;
 126     private NamedArea russia ;
 127     private NamedArea canada ;
 128
 129     @Before
 130     public void setUp() throws Exception {
 131         typesToIndex = new HashSet<Class<? extends CdmBase>>();
 132         typesToIndex.add(DescriptionElementBase.class);
 133         typesToIndex.add(TaxonBase.class);
 134         typesToIndex.add(TaxonRelationship.class);
 135
 136         germany =  Country.GERMANY();
 137         france = Country.FRANCEFRENCHREPUBLIC();
 138         russia = Country.RUSSIANFEDERATION();
 139         canada = Country.CANADA();
 140
 141         includeUnpublished = true;
 142     }
 143
 144     @Test
 145     public void testDbUnitUsageTest() throws Exception {
 146         assertNotNull("taxonService should exist", taxonService);
 147         assertNotNull("nameService should exist", nameService);
 148     }
 149
 150
 151     @SuppressWarnings("rawtypes")
 152     @Test
 153     @DataSet
 154     public final void testPurgeAndReindex() throws IOException, LuceneParseException {
 155
 156         refreshLuceneIndex();
 157
 158         Pager<SearchResult<TaxonBase>> pager;
 159
 160         pager = taxonService.findByFullText(null, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 8
 161         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 162
 163         indexer.purge(null);
 164         commitAndStartNewTransaction(null);
 165
 166         pager = taxonService.findByFullText(null, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 0
 167         Assert.assertEquals("Expecting no entities since the index has been purged", 0, pager.getCount().intValue());
 168
 169         indexer.reindex(indexer.indexedClasses(), null);
 170         commitAndStartNewTransaction(null);
 171
 172         pager = taxonService.findByFullText(null, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 8
 173         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 174     }
 175
 176
 177     @SuppressWarnings("rawtypes")
 178     @Test
 179     @DataSet
 180     public final void testFindByDescriptionElementFullText_CommonName() throws IOException,
 181             LuceneParseException {
 182
 183         refreshLuceneIndex();
 184
 185         Pager<SearchResult<TaxonBase>> pager;
 186
 187         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null, null,
 188                 false, null, null, null, null);
 189         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1,
 190                 pager.getCount().intValue());
 191
 192         // the description containing the Nulltanne has no taxon attached,
 193         // taxon.id = null
 194         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Nulltanne", null, null, null,
 195                 false, null, null, null, null);
 196         Assert.assertEquals("Expecting no entity when searching for 'Nulltanne' ", 0, pager.getCount().intValue());
 197
 198         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 199                 Arrays.asList(new Language[] { Language.GERMAN() }), false, null, null, null, null);
 200         Assert.assertEquals("Expecting one entity when searching in German", 1, pager.getCount().intValue());
 201
 202         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 203                 Arrays.asList(new Language[] { Language.RUSSIAN() }), false, null, null, null, null);
 204         Assert.assertEquals("Expecting no entity when searching in Russian", 0, pager.getCount().intValue());
 205
 206     }
 207
 208     @SuppressWarnings("rawtypes")
 209     @Test
 210     @DataSet
 211     public final void testFindByDescriptionElementFullText_Distribution() throws IOException, LuceneParseException {
 212
 213         refreshLuceneIndex();
 214
 215         Pager<SearchResult<TaxonBase>> pager;
 216         // by Area
 217         pager = taxonService.findByDescriptionElementFullText(null, "Canada", null, null, null, false, null, null, null, null);
 218         Assert.assertEquals("Expecting one entity when searching for arae 'Canada'", 1, pager.getCount().intValue());
 219         // by Status
 220         pager = taxonService.findByDescriptionElementFullText(null, "present", null, null, null, false, null, null, null, null);
 221         Assert.assertEquals("Expecting one entity when searching for status 'present'", 1, pager.getCount().intValue());
 222     }
 223
 224     @SuppressWarnings("rawtypes")
 225     @Test
 226     @DataSet
 227     public final void testFindByDescriptionElementFullText_wildcard() throws IOException, LuceneParseException {
 228
 229         refreshLuceneIndex();
 230
 231         Pager<SearchResult<TaxonBase>> pager;
 232
 233         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
 234         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1, pager.getCount().intValue());
 235     }
 236
 237     /**
 238      * Regression test for #3113 (hibernate search: wildcard query can cause BooleanQuery$TooManyClauses: maxClauseCount is set to 1024)
 239      *
 240      * @throws IOException
 241      * @throws LuceneParseException
 242      */
 243     @SuppressWarnings("rawtypes")
 244     @Test
 245     @DataSet
 246     public final void testFindByDescriptionElementFullText_TooManyClauses() throws IOException, LuceneParseException {
 247
 248         // generate 1024 terms to reproduce the bug
 249         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 250         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 251         while(uniqueRandomStrs.size() < 1024){
 252             uniqueRandomStrs.add(RandomStringUtils.random(10, true, false));
 253         }
 254         for(String rndStr: uniqueRandomStrs){
 255             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 256         }
 257         descriptionService.saveOrUpdate(description);
 258         commitAndStartNewTransaction(null);
 259
 260         refreshLuceneIndex();
 261
 262         Pager<SearchResult<TaxonBase>> pager;
 263
 264         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, null, null, null, null);
 265         Assert.assertEquals("Expecting all 1024 entities grouped into one SearchResult item when searching for Rot*", 1, pager.getCount().intValue());
 266     }
 267
 268     /**
 269      * Regression test for #3116 (fulltext search: always only one page of results)
 270      *
 271      * @throws IOException
 272      * @throws LuceneParseException
 273      */
 274     @SuppressWarnings("rawtypes")
 275     @Test
 276     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 277     @Ignore
 278     public final void testFullText_Paging() throws IOException, LuceneParseException {
 279
 280         Reference sec = ReferenceFactory.newDatabase();
 281         referenceService.save(sec);
 282
 283         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 284         int numOfItems = 100;
 285         while(uniqueRandomStrs.size() < numOfItems){
 286             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 287         }
 288
 289         for(String rndStr: uniqueRandomStrs){
 290
 291             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(Rank.SERIES()), sec);
 292             taxon.setTitleCache("Tax" + rndStr, true);
 293             taxonService.save(taxon);
 294
 295             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 296             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 297             descriptionService.saveOrUpdate(description);
 298         }
 299
 300         commitAndStartNewTransaction(new String[]{"TAXONBASE"});
 301         refreshLuceneIndex();
 302
 303         int pageSize = 10;
 304
 305         Pager<SearchResult<TaxonBase>> pager;
 306
 307         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, null, null, null);
 308         Assert.assertEquals("unexpeted number of pages", Integer.valueOf(numOfItems / pageSize), pager.getPagesAvailable());
 309         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 9, null, null);
 310         Assert.assertNotNull("last page must have records", pager.getRecords());
 311         Assert.assertNotNull("last item on last page must exist", pager.getRecords().get(0));
 312         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 10, null, null);
 313         Assert.assertNotNull("last page + 1 must not have any records", pager.getRecords());
 314     }
 315
 316     /**
 317      * test for max score and sort by score of hit groups
 318      * with all matches per taxon in a single TextData  element
 319      * see {@link #testFullText_ScoreAndOrder_2()} for the complement
 320      * test with matches in multiple TextData per taxon
 321      *
 322      * @throws IOException
 323      * @throws LuceneParseException
 324      */
 325     @SuppressWarnings("rawtypes")
 326     @Test
 327     @DataSet
 328     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 329     public final void testFullText_ScoreAndOrder_1() throws IOException, LuceneParseException {
 330
 331         int numOfTaxa = 3;
 332
 333         UUID[] taxonUuids = new UUID[numOfTaxa];
 334         StringBuilder text = new StringBuilder();
 335
 336         for(int i = 0; i < numOfTaxa; i++){
 337
 338             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 339             taxon.setTitleCache("Taxon_" + i, true);
 340             taxonUuids[i] = taxon.getUuid();
 341             taxonService.save(taxon);
 342
 343             text.append(" ").append("Rot");
 344             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 345             description.addElement(TextData.NewInstance(text.toString(), Language.DEFAULT(), null));
 346             descriptionService.saveOrUpdate(description);
 347         }
 348
 349         commitAndStartNewTransaction(null);
 350         refreshLuceneIndex();
 351
 352         Pager<SearchResult<TaxonBase>> pager;
 353
 354         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 355         for(int i = 0; i < numOfTaxa; i++){
 356             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 357         }
 358         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 359     }
 360
 361     /**
 362      * test for max score and sort by score of hit groups
 363      * with all matches per taxon in a multiple TextData elements
 364      * see {@link #testFullText_ScoreAndOrder_1()} for the complement
 365      * test with matches in a single TextData per taxon
 366      *
 367      * @throws IOException
 368      * @throws LuceneParseException
 369      */
 370     @SuppressWarnings("rawtypes")
 371     @Test
 372     @DataSet
 373     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 374     public final void testFullText_ScoreAndOrder_2() throws IOException, LuceneParseException {
 375
 376         int numOfTaxa = 3;
 377
 378         UUID[] taxonUuids = new UUID[numOfTaxa];
 379
 380         for(int i = 0; i < numOfTaxa; i++){
 381
 382             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 383             taxon.setTitleCache("Taxon_" + i, true);
 384             taxonUuids[i] = taxon.getUuid();
 385             taxonService.save(taxon);
 386
 387             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 388             for(int k = 0; k < i; k++){
 389                 description.addElement(TextData.NewInstance("Rot", Language.DEFAULT(), null));
 390             }
 391             descriptionService.saveOrUpdate(description);
 392         }
 393
 394         commitAndStartNewTransaction(null);
 395         refreshLuceneIndex();
 396
 397         Pager<SearchResult<TaxonBase>> pager;
 398
 399         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 400         for(int i = 0; i < numOfTaxa; i++){
 401             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 402         }
 403         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 404     }
 405
 406
 407     /**
 408      * @throws IOException
 409      * @throws LuceneParseException
 410      * @throws LuceneMultiSearchException
 411      */
 412     @Test
 413     @DataSet
 414     public final void testFullText_Grouping() throws IOException, LuceneParseException, LuceneMultiSearchException {
 415
 416         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 417         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 418         int numOfItems = 100;
 419         while(uniqueRandomStrs.size() < numOfItems){
 420             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 421         }
 422         for(String rndStr: uniqueRandomStrs){
 423             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 424         }
 425         descriptionService.saveOrUpdate(description);
 426
 427         commitAndStartNewTransaction(new String[]{"DESCRIPTIONELEMENTBASE"});
 428
 429         refreshLuceneIndex();
 430
 431         int pageSize = 10;
 432
 433         Pager<SearchResult<TaxonBase>> pager;
 434         boolean highlightFragments = true;
 435
 436         // test with findByDescriptionElementFullText
 437         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, highlightFragments, pageSize, null, null, null);
 438         logFreeTextSearchResults(pager, Level.DEBUG, null);
 439         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 440         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 441         Map<String, String[]> highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 442         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 443         int maxDocsPerGroup = 10;
 444         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 445
 446         // test with findByEverythingFullText
 447         pager = taxonService.findByEverythingFullText( "Rot*", null, includeUnpublished, null, highlightFragments, pageSize, null, null, null);
 448         logFreeTextSearchResults(pager, Level.DEBUG, null);
 449         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 450         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 451         highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 452         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 453         maxDocsPerGroup = 10;
 454         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 455
 456     }
 457
 458     @SuppressWarnings("rawtypes")
 459     @Test
 460     @DataSet
 461     @Ignore
 462     public final void testFindByDescriptionElementFullText_TextData() throws IOException, LuceneParseException {
 463
 464         refreshLuceneIndex();
 465
 466         Pager<SearchResult<TaxonBase>> pager;
 467         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, false, null, null, null, null);
 468         logFreeTextSearchResults(pager, Level.DEBUG, null);
 469         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 470         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 471         Assert.assertTrue("Expecting two docs, one for RUSSIAN and one for GERMAN", pager.getRecords().get(0).getDocs().size() == 2);
 472         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 473
 474
 475         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, null, null, false, null, null, null, null);
 476         Assert.assertEquals("Expecting one entity when searching for any type", 1, pager.getCount().intValue());
 477
 478         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.UNKNOWN()}), null, false, null, null, null, null);
 479         Assert.assertEquals("Expecting one entity when searching for any type and for Feature DESCRIPTION", 1, pager.getCount().intValue());
 480
 481         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER()}), null, false, null, null, null, null);
 482         Assert.assertEquals("Expecting no entity when searching for any type and for Feature CHROMOSOME_NUMBER", 0, pager.getCount().intValue());
 483
 484         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER(), Feature.UNKNOWN()}), null, false, null, null, null, null);
 485         Assert.assertEquals("Expecting no entity when searching for any type and for Feature DESCRIPTION or CHROMOSOME_NUMBER", 1, pager.getCount().intValue());
 486
 487         pager = taxonService.findByDescriptionElementFullText(Distribution.class, "Abies", null, null, null, false, null, null, null, null);
 488         Assert.assertEquals("Expecting no entity when searching for Distribution", 0, pager.getCount().intValue());
 489
 490         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{}), false, null, null, null, null);
 491         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 492         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 493
 494         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.RUSSIAN()}), false, null, null, null, null);
 495         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 496         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 497
 498         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 499         Assert.assertEquals("Expecting no entity", 0, pager.getCount().intValue());
 500
 501         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 502         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 503         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 504     }
 505
 506     @SuppressWarnings("rawtypes")
 507     @Test
 508     @DataSet
 509     public final void testFindByDescriptionElementFullText_MultipleWords() throws IOException, LuceneParseException {
 510
 511         refreshLuceneIndex();
 512
 513         // Pflanzenart aus der Gattung der Tannen
 514         long start = System.currentTimeMillis();
 515
 516         Pager<SearchResult<TaxonBase>> pager;
 517         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, false, null, null, null, null);
 518         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 519
 520         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Wespen", null, null, null, false, null, null, null, null);
 521         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 522
 523         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, false, null, null, null, null);
 524         Assert.assertEquals("AND search : Expecting one entity", 1, pager.getCount().intValue());
 525
 526         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Wespen", null, null, null, false, null, null, null, null);
 527         Assert.assertEquals("AND search : Expecting no entity", 0, pager.getCount().intValue());
 528
 529         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, false, null, null, null, null);
 530         Assert.assertEquals("Phrase search : Expecting one entity", 1, pager.getCount().intValue());
 531
 532         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Wespen\"", null, null, null, false, null, null, null, null);
 533         Assert.assertEquals("Phrase search : Expecting one entity", 0, pager.getCount().intValue());
 534
 535         logger.info("testFindByDescriptionElementFullText_MultipleWords() duration: " + (System.currentTimeMillis() - start) + "ms");
 536
 537     }
 538
 539
 540     @SuppressWarnings("rawtypes")
 541     @Test
 542     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 543     public final void testFindByDescriptionElementFullText_modify_DescriptionElement() throws IOException, LuceneParseException {
 544
 545         refreshLuceneIndex();
 546
 547         Pager<SearchResult<TaxonBase>> pager;
 548         //
 549         // modify the DescriptionElement
 550         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 551         Assert.assertTrue("Search did not return any results", pager.getRecords().size() > 0);
 552         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 553         Document indexDocument = pager.getRecords().get(0).getDocs().iterator().next();
 554         String[] descriptionElementUuidStr = indexDocument.getValues("uuid");
 555         String[] inDescriptionUuidStr = indexDocument.getValues("inDescription.uuid");
 556         // is only one uuid!
 557         DescriptionElementBase textData = descriptionService.getDescriptionElementByUuid(UUID.fromString(descriptionElementUuidStr[0]));
 558
 559         ((TextData)textData).removeText(Language.GERMAN());
 560         ((TextData)textData).putText(Language.SPANISH_CASTILIAN(), "abeto bals"+UTF8.SMALL_A_ACUTE+"mico");
 561
 562         descriptionService.saveDescriptionElement(textData);
 563         commitAndStartNewTransaction(null);
 564 //        printDataSet(System.out, new String[] {
 565 //                "DESCRIPTIONELEMENTBASE", "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING" }
 566 //        );
 567
 568         //
 569         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 570         Assert.assertEquals("The german 'Balsam-Tanne' TextData should no longer be indexed", 0, pager.getCount().intValue());
 571         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 572         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 573         pager = taxonService.findByDescriptionElementFullText(TextData.class, "bals"+UTF8.SMALL_A_ACUTE+"mico", null, null, null, false, null, null, null, null);
 574         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 575
 576         //
 577         // modify the DescriptionElement via the Description object
 578         DescriptionBase<?> description = descriptionService.find(UUID.fromString(inDescriptionUuidStr[0]));
 579         Set<DescriptionElementBase> elements = description.getElements();
 580         for( DescriptionElementBase elm : elements){
 581             if(elm.getUuid().toString().equals(descriptionElementUuidStr[0])){
 582                 ((TextData)elm).removeText(Language.SPANISH_CASTILIAN());
 583                 ((TextData)elm).putText(Language.POLISH(), "Jod"+UTF8.POLISH_L+"a balsamiczna");
 584             }
 585         }
 586         descriptionService.saveOrUpdate(description);
 587         commitAndStartNewTransaction(null);
 588         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 589         Assert.assertEquals("The spanish 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico' TextData should no longer be indexed", 0, pager.getCount().intValue());
 590         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsamiczna", null, null, Arrays.asList(new Language[]{Language.POLISH()}), false, null, null, null, null);
 591         Assert.assertEquals("expecting to find the POLISH 'Jod"+UTF8.POLISH_L+"a balsamiczna'", 1, pager.getCount().intValue());
 592     }
 593
 594     @SuppressWarnings("rawtypes")
 595     @Test
 596     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 597     public final void testFindByDescriptionElementFullText_modify_Taxon() throws IOException, LuceneParseException {
 598
 599         refreshLuceneIndex();
 600
 601         Pager<SearchResult<TaxonBase>> pager;
 602         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
 603         TaxonDescription d_abies_balsamea = (TaxonDescription)descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 604
 605         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 606         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne'", 1, pager.getCount().intValue());
 607
 608         // exchange the Taxon with another one via the Taxon object
 609         // 1.) remove existing description:
 610         t_abies_balsamea.removeDescription(d_abies_balsamea);
 611
 612         taxonService.saveOrUpdate(t_abies_balsamea);
 613         commitAndStartNewTransaction(null);
 614
 615         t_abies_balsamea = (Taxon)taxonService.find(t_abies_balsamea.getUuid());
 616
 617         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 618         Assert.assertEquals("'Balsam-Tanne' should no longer be found", 0, pager.getCount().intValue());
 619
 620         // 2.) create new description and add to taxon:
 621         TaxonDescription d_abies_balsamea_new = TaxonDescription.NewInstance();
 622         d_abies_balsamea_new
 623                 .addElement(TextData
 624                         .NewInstance(
 625                                 "Die Balsamtanne ist mit bis zu 30 m Höhe ein mittelgro"+UTF8.SHARP_S+"er Baum und kann bis zu 200 Jahre alt werden",
 626                                 Language.GERMAN(), null));
 627         t_abies_balsamea.addDescription(d_abies_balsamea_new);
 628         // set authorshipCache to null to avoid validation exception,
 629         // this is maybe not needed in future,  see ticket #3344
 630         IBotanicalName abies_balsamea = CdmBase.deproxy(t_abies_balsamea.getName(), TaxonName.class);
 631         abies_balsamea.setAuthorshipCache(null);
 632         printDataSet(System.err, new String[] {"LANGUAGESTRING_AUD"});
 633         taxonService.saveOrUpdate(t_abies_balsamea);
 634         commitAndStartNewTransaction(null);
 635
 636 //        printDataSet(System.out, new String[] {
 637 //                "DESCRIPTIONBASE"
 638 //        });
 639
 640         pager = taxonService.findByDescriptionElementFullText(TextData.class, "mittelgro"+UTF8.SHARP_S+"er Baum", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 641         Assert.assertEquals("the taxon should be found via the new Description", 1, pager.getCount().intValue());
 642     }
 643
 644     @SuppressWarnings("rawtypes")
 645     @Test
 646     @DataSet
 647     public final void testFindByDescriptionElementFullText_modify_Classification() throws IOException, LuceneParseException {
 648
 649         refreshLuceneIndex();
 650
 651         Pager<SearchResult<TaxonBase>> pager;
 652
 653         // put taxon into other classification, new taxon node
 654         Classification classification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 655         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 656
 657         // TODO: why is the test failing when the childNode is already retrieved here, and not after the following four lines?
 658         //TaxonNode childNode = classification.getChildNodes().iterator().next();
 659
 660         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 661         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne' even if filtering by classification", 1, pager.getCount().intValue());
 662         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 663         Assert.assertEquals("GERMAN 'Balsam-Tanne' should NOT be found in other classification", 0, pager.getCount().intValue());
 664
 665         // check for the right taxon node
 666         TaxonNode childNode = classification.getChildNodes().iterator().next();
 667         Assert.assertEquals("expecting Abies balsamea sec.", childNode.getTaxon().getUuid().toString(), ABIES_BALSAMEA_UUID);
 668         Assert.assertEquals("expecting default classification", childNode.getClassification().getUuid().toString(), CLASSIFICATION_UUID);
 669
 670         // moving the taxon around, the rootnode is only a proxy
 671         alternateClassification.setRootNode(HibernateProxyHelper.deproxy(alternateClassification.getRootNode(), TaxonNode.class));
 672         alternateClassification.addChildNode(childNode, null, null);
 673
 674         classificationService.saveOrUpdate(alternateClassification);
 675         commitAndStartNewTransaction(null);
 676
 677 //        printDataSet(System.out, new String[] {
 678 //            "TAXONBASE", "TAXONNODE", "CLASSIFICATION"
 679 //        });
 680
 681         // reload classification
 682         classification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 683         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 684         Assert.assertEquals("GERMAN 'Balsam-Tanne' should now be found in other classification", 1, pager.getCount().intValue());
 685
 686         classification.getChildNodes().clear();
 687         classificationService.saveOrUpdate(classification);
 688         commitAndStartNewTransaction(null);
 689
 690         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", classification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 691         Assert.assertEquals("Now the GERMAN 'Balsam-Tanne' should NOT be found in original classification", 0, pager.getCount().intValue());
 692
 693     }
 694
 695     @SuppressWarnings("rawtypes")
 696     @Test
 697     @DataSet
 698     public final void testFindByDescriptionElementFullText_CategoricalData() throws IOException, LuceneParseException {
 699
 700         // add CategoricalData
 701         DescriptionBase d_abies_balsamea = descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 702         // Categorical data
 703         CategoricalData cdata = CategoricalData.NewInstance();
 704         cdata.setFeature(Feature.DESCRIPTION());
 705         State state = State.NewInstance("green", "green", "gn");
 706
 707         StateData statedata = StateData.NewInstance(state);
 708         statedata.putModifyingText(Language.ENGLISH(), "always, even during winter");
 709         cdata.addStateData(statedata);
 710         d_abies_balsamea.addElement(cdata);
 711
 712         UUID termUUID = termService.save(state).getUuid();
 713         descriptionService.save(d_abies_balsamea);
 714
 715         commitAndStartNewTransaction(null);
 716
 717 //        printDataSet(System.out, new String[] {
 718 //                 "STATEDATA", "STATEDATA_DEFINEDTERMBASE", "STATEDATA_LANGUAGESTRING", "LANGUAGESTRING"});
 719
 720         refreshLuceneIndex();
 721
 722         Pager<SearchResult<TaxonBase>> pager;
 723         pager = taxonService.findByDescriptionElementFullText(CategoricalData.class, "green", null, null, null, false, null, null, null, null);
 724         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 725         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 726         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 727         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 728
 729
 730         //TODO modify the StateData
 731         TaxonBase taxon = pager.getRecords().get(0).getEntity();
 732
 733         String newName = "Quercus robur";
 734         taxon.setTitleCache(newName + " sec. ", true);
 735
 736         taxonService.saveOrUpdate(taxon);
 737         commitAndStartNewTransaction(null);
 738
 739         taxon = taxonService.find(taxon.getUuid());
 740         Assert.assertEquals(newName + " sec. ", taxon.getTitleCache());
 741         DefinedTermBase term = termService.find(termUUID);
 742
 743         termService.delete(term);
 744
 745     }
 746
 747     @SuppressWarnings("rawtypes")
 748     @Test
 749     @DataSet
 750     public final void testFindByDescriptionElementFullText_Highlighting() throws IOException, LuceneParseException {
 751
 752         refreshLuceneIndex();
 753
 754         Pager<SearchResult<TaxonBase>> pager;
 755         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, true, null, null, null, null);
 756         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 757         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 758         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
 759         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 760         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Abies</B>"));
 761
 762         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, true, null, null, null, null);
 763         searchResult = pager.getRecords().get(0);
 764         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 765         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 766         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") || fragments[0].contains("<B>Tannen</B>"));
 767
 768         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, true, null, null, null, null);
 769         searchResult = pager.getRecords().get(0);
 770         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 771         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 772         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") && fragments[0].contains("<B>Tannen</B>"));
 773
 774         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, true, null, null, null, null);
 775         searchResult = pager.getRecords().get(0);
 776         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 777         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 778         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B> <B>aus</B> <B>der</B> <B>Gattung</B> <B>der</B> <B>Tannen</B>"));
 779
 780         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Gatt*", null, null, null, true, null, null, null, null);
 781         searchResult = pager.getRecords().get(0);
 782         Assert.assertTrue("Wildcard search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 783         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 784         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Gatt"));
 785     }
 786
 787
 788     @Test
 789     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 790     public final void testFindByFullText() throws IOException, LuceneParseException {
 791
 792         refreshLuceneIndex();
 793
 794         classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 795
 796         Pager<SearchResult<TaxonBase>> pager;
 797         boolean NO_UNPUBLISHED = false;
 798
 799         pager = taxonService.findByFullText(null, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 7
 800 //        logFreeTextSearchResults(pager, Level.DEBUG, null);
 801         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 802
 803         pager = taxonService.findByFullText(null, "Abies", null, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 804 //        logFreeTextSearchResults(pager, Level.DEBUG, null);
 805         Assert.assertEquals("Expecting 6 entities", 6, pager.getCount().intValue());
 806
 807
 808         pager = taxonService.findByFullText(Taxon.class, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 6
 809         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 810
 811         pager = taxonService.findByFullText(Synonym.class, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 1
 812         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 813         pager = taxonService.findByFullText(Synonym.class, "Abies", null, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 1
 814         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 815
 816         pager = taxonService.findByFullText(TaxonBase.class, "sec", null, includeUnpublished, null, true, null, null, null, null); // --> 7
 817         Assert.assertEquals("Expecting 8 entities", 9, pager.getCount().intValue());
 818
 819         pager = taxonService.findByFullText(null, "genus", null, includeUnpublished, null, true, null, null, null, null); // --> 1
 820         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 821
 822         pager = taxonService.findByFullText(Taxon.class, "subalpina", null, includeUnpublished, null, true, null, null, null, null); // --> 0
 823         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 824
 825         // synonym in classification ???
 826     }
 827
 828     @Test
 829     @DataSet
 830     public final void testPrepareByAreaSearch() throws IOException, LuceneParseException {
 831
 832         List<PresenceAbsenceTerm> statusFilter = new ArrayList<PresenceAbsenceTerm>();
 833         List<NamedArea> areaFilter = new ArrayList<NamedArea>();
 834         areaFilter.add(germany);
 835         areaFilter.add(canada);
 836         areaFilter.add(russia);
 837
 838         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDistribution(areaFilter, statusFilter, null, 20, 0, null, null);
 839         Assert.assertEquals("Expecting 2 entities", Integer.valueOf(2), Integer.valueOf(pager.getRecords().size()));
 840
 841     }
 842
 843     @Test
 844     @DataSet
 845     public final void testFindTaxaAndNamesByFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
 846
 847         refreshLuceneIndex();
 848
 849         Pager<SearchResult<TaxonBase>> pager;
 850
 851         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 852
 853
 854         pager = taxonService.findTaxaAndNamesByFullText(
 855                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 856                 "Abies", null, null, null, null, true, null, null, null, null);
 857 //        logPagerRecords(pager, Level.DEBUG);
 858         Assert.assertEquals("doTaxa & doSynonyms & unpublished", 8, pager.getCount().intValue());
 859         pager = taxonService.findTaxaAndNamesByFullText(
 860                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 861                 "Abies", null, null, null, null, true, null, null, null, null);
 862         Assert.assertEquals("doTaxa & doSynonyms", 6, pager.getCount().intValue());
 863
 864         EnumSet<TaxaAndNamesSearchMode> searchMode = EnumSet.allOf(TaxaAndNamesSearchMode.class);
 865         pager = taxonService.findTaxaAndNamesByFullText(
 866                 searchMode,
 867                 "Abies", null, null, null, null, true, null, null, null, null);
 868 //        logPagerRecords(pager, Level.DEBUG);
 869         Assert.assertEquals("all search modes", 8, pager.getCount().intValue());
 870         searchMode.remove(TaxaAndNamesSearchMode.includeUnpublished);
 871         pager = taxonService.findTaxaAndNamesByFullText(
 872                 searchMode,
 873                 "Abies", null, null, null, null, true, null, null, null, null);
 874         Assert.assertEquals("all search modes except unpublished", 6, pager.getCount().intValue());
 875
 876         pager = taxonService.findTaxaAndNamesByFullText(
 877                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 878                 "Abies", alternateClassification, null, null, null, true, null, null, null, null);
 879 //        logPagerRecords(pager, Level.DEBUG);
 880         Assert.assertEquals("all search modes, filtered by alternateClassification", 1, pager.getCount().intValue());
 881
 882         pager = taxonService.findTaxaAndNamesByFullText(
 883                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 884                 "Abies", null, null, null, null, true, null, null, null, null);
 885         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 886         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 887         Assert.assertEquals(Synonym.class, searchResult.getEntity().getClass());
 888         // Abies subalpina sec. Kohlbecker, A., Testcase standart views, 2013
 889
 890
 891         pager = taxonService.findTaxaAndNamesByFullText(
 892                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
 893                 "Abies", null, null, null, null, true, null, null, null, null);
 894         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 895
 896
 897         pager = taxonService.findTaxaAndNamesByFullText(
 898                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
 899                 "Tanne", null, null, null, null, true, null, null, null, null);
 900         Assert.assertEquals("Expecting 1 entity", 1, pager.getRecords().size());
 901         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 902
 903         pager = taxonService.findTaxaAndNamesByFullText(
 904                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
 905                 "kawakamii", (Classification)null, null, null, null, true, null, null, null, null);
 906         logFreeTextSearchResults(pager, Level.DEBUG, null);
 907         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 908         searchResult = pager.getRecords().get(0);
 909
 910     }
 911
 912     @Test
 913     @DataSet
 914     public final void testFindTaxaAndNamesByFullText_PhraseQuery() throws IOException, LuceneParseException, LuceneMultiSearchException {
 915
 916         refreshLuceneIndex();
 917
 918         Pager<SearchResult<TaxonBase>> pager;
 919
 920
 921         pager = taxonService.findTaxaAndNamesByFullText(
 922                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 923                 "\"Abies alba\"", null, null, null, null, true, null, null, null, null);
 924 //        logPagerRecords(pager, Level.DEBUG);
 925         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 1, pager.getCount().intValue());
 926
 927         pager = taxonService.findTaxaAndNamesByFullText(
 928                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 929                 "\"Abies al*\"", null, null, null, null, true, null, null, null, null);
 930 //        logPagerRecords(pager, Level.DEBUG);
 931         Assert.assertEquals("doTaxa & doSynonyms with complex phrase query", 1, pager.getCount().intValue());
 932
 933         pager = taxonService.findTaxaAndNamesByFullText(
 934                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 935                 "\"Abies*\"", null, null, null, null, true, null, null, null, null);
 936 //        logPagerRecords(pager, Level.DEBUG);
 937         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 8, pager.getCount().intValue());
 938
 939     }
 940
 941     @Test
 942     @DataSet
 943     public final void testFindTaxaAndNamesByFullText_Sort() throws IOException, LuceneParseException, LuceneMultiSearchException {
 944
 945         refreshLuceneIndex();
 946
 947         Pager<SearchResult<TaxonBase>> pager;
 948
 949         List<OrderHint> orderHints = new ArrayList<OrderHint>();
 950
 951         String[] docFields2log = new String[]{"id"};
 952
 953         // SortById
 954         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
 955         pager = taxonService.findTaxaAndNamesByFullText(
 956                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
 957                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 958 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 959         int lastId = -1;
 960         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 961             if(lastId != -1){
 962                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
 963             }
 964             lastId = rs.getEntity().getId();
 965         }
 966
 967         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
 968         pager = taxonService.findTaxaAndNamesByFullText(
 969                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 970                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 971 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 972
 973         lastId = -1;
 974         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 975             if(lastId != -1){
 976                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
 977             }
 978             lastId = rs.getEntity().getId();
 979         }
 980
 981         // Sortby NOMENCLATURAL_SORT_ORDER TODO make assertions !!!
 982         orderHints.clear();
 983         orderHints.addAll(OrderHint.NOMENCLATURAL_SORT_ORDER.asList());
 984         pager = taxonService.findTaxaAndNamesByFullText(
 985                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 986                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 987         logFreeTextSearchResults(pager, Level.DEBUG, null);
 988
 989     }
 990
 991     @Test
 992     @DataSet
 993     public final void testFindTaxaAndNamesByFullText_AreaFilter() throws IOException, LuceneParseException, LuceneMultiSearchException {
 994
 995         refreshLuceneIndex();
 996
 997         Pager<SearchResult<TaxonBase>> pager;
 998
 999         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1000         a_germany_canada_russia.add(germany);
1001         a_germany_canada_russia.add(canada);
1002         a_germany_canada_russia.add(russia);
1003
1004         Set<NamedArea> a_russia = new HashSet<>();
1005         a_russia.add(russia);
1006
1007         Set<PresenceAbsenceTerm> present = new HashSet<>();
1008         present.add(PresenceAbsenceTerm.PRESENT());
1009
1010         Set<PresenceAbsenceTerm> present_native = new HashSet<>();
1011         present_native.add(PresenceAbsenceTerm.PRESENT());
1012         present_native.add(PresenceAbsenceTerm.NATIVE());
1013
1014         Set<PresenceAbsenceTerm> absent = new HashSet<>();
1015         absent.add(PresenceAbsenceTerm.ABSENT());
1016
1017         pager = taxonService.findTaxaAndNamesByFullText(
1018                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished),
1019                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1020         logFreeTextSearchResults(pager, Level.DEBUG, null);
1021
1022         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1023         pager = taxonService.findTaxaAndNamesByFullText(
1024                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1025                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1026         Assert.assertEquals("synonyms with matching area filter", 1, pager.getCount().intValue());
1027
1028         pager = taxonService.findTaxaAndNamesByFullText(
1029                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1030                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1031         logFreeTextSearchResults(pager, Level.DEBUG, null);
1032         Assert.assertEquals("taxa and synonyms with matching area filter", 3, pager.getCount().intValue());
1033
1034         pager = taxonService.findTaxaAndNamesByFullText(
1035                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1036                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1037         Assert.assertEquals("taxa and synonyms with matching area & status filter 1", 3, pager.getCount().intValue());
1038
1039         pager = taxonService.findTaxaAndNamesByFullText(
1040                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1041                 "Abies", null, a_germany_canada_russia, present, null, true, null, null, null, null);
1042         Assert.assertEquals("taxa and synonyms with matching area & status filter 2", 2, pager.getCount().intValue());
1043
1044         pager = taxonService.findTaxaAndNamesByFullText(
1045                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1046                 "Abies", null, a_russia, present, null, true, null, null, null, null);
1047         Assert.assertEquals("taxa and synonyms with non matching area & status filter", 0, pager.getCount().intValue());
1048
1049         pager = taxonService.findTaxaAndNamesByFullText(
1050                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
1051                 "Tanne", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1052         Assert.assertEquals("ByCommonNames with area filter", 1, pager.getCount().intValue());
1053
1054         // abies_kawakamii_sensu_komarov as misapplied name for t_abies_balsamea
1055         pager = taxonService.findTaxaAndNamesByFullText(
1056                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1057                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1058         Assert.assertEquals("misappliedNames with matching area & status filter", 1, pager.getCount().intValue());
1059
1060
1061         // 1. remove existing taxon relation
1062         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
1063         Set<TaxonRelationship> relsTo = t_abies_balsamea.getRelationsToThisTaxon();
1064         Assert.assertEquals(1, relsTo.size());
1065         TaxonRelationship taxonRelation = relsTo.iterator().next();
1066         t_abies_balsamea.removeTaxonRelation(taxonRelation);
1067         taxonService.saveOrUpdate(t_abies_balsamea);
1068         commitAndStartNewTransaction(null);
1069
1070         pager = taxonService.findTaxaAndNamesByFullText(
1071                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1072                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1073         Assert.assertEquals("misappliedNames with matching area & status filter, should match nothing now", 0, pager.getCount().intValue());
1074
1075         // 2. now add abies_kawakamii_sensu_komarov as misapplied name for t_abies_alba and search for misapplications in russia: ABSENT
1076         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(UUID.fromString(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1077         Taxon t_abies_alba = (Taxon)taxonService.find(UUID.fromString(ABIES_ALBA_UUID));
1078         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1079
1080         taxonService.update(t_abies_kawakamii_sensu_komarov);
1081
1082         commitAndStartNewTransaction(null);
1083
1084         pager = taxonService.findTaxaAndNamesByFullText(
1085                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1086                 "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1087         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1088
1089     }
1090
1091     @Test
1092     @DataSet
1093     @Ignore // remove once http://dev.e-taxonomy.eu/trac/ticket/5477 is solved
1094     public final void testFindTaxaAndNamesByFullText_AreaFilter_issue5477() throws IOException, LuceneParseException, LuceneMultiSearchException {
1095
1096         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1097         a_germany_canada_russia.add(germany);
1098         a_germany_canada_russia.add(canada);
1099         a_germany_canada_russia.add(russia);
1100
1101
1102         Set<PresenceAbsenceTerm> absent = new HashSet<PresenceAbsenceTerm>();
1103         absent.add(PresenceAbsenceTerm.ABSENT());
1104
1105         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(UUID.fromString(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1106         Taxon t_abies_alba = (Taxon)taxonService.find(UUID.fromString(ABIES_ALBA_UUID));
1107         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1108
1109         /* Since the upgrade from hibernate search 4 to 5.5
1110          * triggering an update of t_abies_alba is no longer sufficient to also update the
1111          * document of t_abies_kawakamii_sensu_komarov in the lucene index.
1112          * the last test in testFindTaxaAndNamesByFullText_AreaFilter() failed in this case.
1113          * This situation is reproduced here:
1114          */
1115         taxonService.update(t_abies_alba);
1116
1117           commitAndStartNewTransaction(null);
1118
1119           Pager pager = taxonService.findTaxaAndNamesByFullText(
1120                   EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1121                   "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1122           Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1123     }
1124
1125
1126     /**
1127      * Regression test for #3119: fulltext search: Entity always null whatever search
1128      *
1129      * @throws IOException
1130      * @throws LuceneParseException
1131      * @throws LuceneMultiSearchException
1132      */
1133     @Test
1134     @DataSet
1135     public final void testFindByEverythingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1136
1137         refreshLuceneIndex();
1138
1139         Pager<SearchResult<TaxonBase>> pager;
1140
1141         // via Taxon
1142         pager = taxonService.findByEverythingFullText("Abies", null, includeUnpublished, null, true, null, null, null, null);
1143         logFreeTextSearchResults(pager, Level.DEBUG, null);
1144         Assert.assertTrue("Expecting at least 7 entities for 'Abies'", pager.getCount() > 7);
1145         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1146         Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1147
1148         // via DescriptionElement
1149         pager = taxonService.findByEverythingFullText("present", null, includeUnpublished, null, true, null, null, null, null);
1150         Assert.assertEquals("Expecting one entity when searching for area 'present'", 1, pager.getCount().intValue());
1151         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1152         Assert.assertEquals("Expecting Taxon entity", Taxon.class, CdmBase.deproxy(pager.getRecords().get(0).getEntity()).getClass());
1153         Assert.assertEquals("Expecting Taxon ", ABIES_BALSAMEA_UUID, pager.getRecords().get(0).getEntity().getUuid().toString());
1154
1155     }
1156
1157
1158     @Test
1159     @DataSet
1160     public final void findByEveryThingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1161
1162         refreshLuceneIndex();
1163
1164         Pager<SearchResult<TaxonBase>> pager;
1165
1166         pager = taxonService.findByEverythingFullText("genus", null, includeUnpublished, null, false, null, null, null, null); // --> 1
1167         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1168
1169         //FIXME FAILS: abies balamea is returned twice, see also testFullText_Grouping()
1170         pager = taxonService.findByEverythingFullText("Balsam", null, includeUnpublished, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1171         logFreeTextSearchResults(pager, Level.DEBUG, null);
1172         Assert.assertEquals("expecting to find the Abies balsamea via the GERMAN DescriptionElements", 1, pager.getCount().intValue());
1173
1174         pager = taxonService.findByEverythingFullText("Abies", null, includeUnpublished, null, true, null, null, null, null);
1175         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
1176         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
1177         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
1178         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
1179         Assert.assertTrue("first fragments should contains serch term", fragments[0].toLowerCase().contains("<b>abies</b>"));
1180     }
1181
1182 //    @SuppressWarnings("rawtypes")
1183 //    @Test
1184 //    @DataSet
1185 //    public final void benchmarkFindTaxaAndNamesHql() throws IOException, LuceneParseException {
1186 //
1187 //        createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1188 //
1189 //        IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1190 //        configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1191 //        configurator.setMatchMode(MatchMode.BEGINNING);
1192 //        configurator.setDoTaxa(false);
1193 //        configurator.setDoSynonyms(false);
1194 //        configurator.setDoNamesWithoutTaxa(false);
1195 //        configurator.setDoTaxaByCommonNames(true);
1196 //
1197 //        Pager<IdentifiableEntity> pager;
1198 //
1199 //        long startMillis = System.currentTimeMillis();
1200 //        for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1201 //            pager = taxonService.findTaxaAndNames(configurator);
1202 //            if (logger.isDebugEnabled()) {
1203 //                logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1204 //            }
1205 //        }
1206 //        double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1207 //        logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1208 //    }
1209
1210     @SuppressWarnings("rawtypes")
1211     @Test
1212     @DataSet
1213     public final void benchmarkFindByCommonNameHql() {
1214
1215 //        printDataSet(System.err, new String[] { "TaxonBase" });
1216
1217         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1218
1219         IFindTaxaAndNamesConfigurator configurator = FindTaxaAndNamesConfiguratorImpl.NewInstance();
1220         configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1221         configurator.setMatchMode(MatchMode.BEGINNING);
1222         configurator.setDoTaxa(false);
1223         configurator.setDoSynonyms(false);
1224         configurator.setDoNamesWithoutTaxa(false);
1225         configurator.setDoTaxaByCommonNames(true);
1226
1227         Pager<IdentifiableEntity> pager;
1228
1229         long startMillis = System.currentTimeMillis();
1230         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1231             pager = taxonService.findTaxaAndNames(configurator);
1232             if (logger.isDebugEnabled()) {
1233                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1234             }
1235         }
1236         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1237         logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1238     }
1239
1240     @SuppressWarnings("rawtypes")
1241     @Test
1242     @DataSet
1243     public final void benchmarkFindByCommonNameLucene() throws IOException, LuceneParseException {
1244
1245         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1246
1247         refreshLuceneIndex();
1248
1249         Pager<SearchResult<TaxonBase>> pager;
1250
1251         long startMillis = System.currentTimeMillis();
1252         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1253             pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
1254             if (logger.isDebugEnabled()) {
1255                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getEntity().getTitleCache());
1256             }
1257         }
1258         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1259         logger.info("Benchmark result - [find taxon by CommonName via lucene] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1260     }
1261
1262     /**
1263      * uncomment @Test annotation to create the dataset for this test
1264      */
1265     @Override
1266     //    @Test
1267     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="BlankDataSet.xml")
1268     public final void createTestDataSet() throws FileNotFoundException {
1269
1270         Classification europeanAbiesClassification = Classification.NewInstance("European Abies");
1271         europeanAbiesClassification.setUuid(UUID.fromString(CLASSIFICATION_UUID));
1272         classificationService.save(europeanAbiesClassification);
1273
1274         Classification alternativeClassification = Classification.NewInstance("Abies alternative");
1275         alternativeClassification.setUuid(UUID.fromString(CLASSIFICATION_ALT_UUID));
1276         classificationService.save(alternativeClassification);
1277
1278         Reference sec = ReferenceFactory.newBook();
1279         sec.setTitleCache("Kohlbecker, A., Testcase standart views, 2013", true);
1280         Reference sec_sensu = ReferenceFactory.newBook();
1281         sec_sensu.setTitleCache("Komarov, V. L., Flora SSSR 29", true);
1282         referenceService.save(sec);
1283         referenceService.save(sec_sensu);
1284
1285         IBotanicalName n_abies = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
1286         n_abies.setNameCache("Abies", true);
1287         Taxon t_abies = Taxon.NewInstance(n_abies, sec);
1288         taxonService.save(t_abies);
1289
1290         IBotanicalName n_abies_alba = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1291         n_abies_alba.setNameCache("Abies alba", true);
1292         Taxon t_abies_alba = Taxon.NewInstance(n_abies_alba, sec);
1293         t_abies_alba.setUuid(UUID.fromString(ABIES_ALBA_UUID));
1294         taxonService.save(t_abies_alba);
1295
1296         IBotanicalName n_abies_subalpina = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1297         n_abies_subalpina.setNameCache("Abies subalpina", true);
1298         Synonym s_abies_subalpina = Synonym.NewInstance(n_abies_subalpina, sec);
1299         taxonService.save(s_abies_subalpina);
1300
1301         IBotanicalName n_abies_balsamea = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1302         n_abies_balsamea.setNameCache("Abies balsamea", true);
1303         Taxon t_abies_balsamea = Taxon.NewInstance(n_abies_balsamea, sec);
1304         t_abies_balsamea.setUuid(UUID.fromString(ABIES_BALSAMEA_UUID));
1305         t_abies_balsamea.addSynonym(s_abies_subalpina, SynonymType.SYNONYM_OF());
1306         taxonService.save(t_abies_balsamea);
1307
1308         IBotanicalName n_abies_grandis = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1309         n_abies_grandis.setNameCache("Abies grandis", true);
1310         Taxon t_abies_grandis = Taxon.NewInstance(n_abies_grandis, sec);
1311         taxonService.save(t_abies_grandis);
1312
1313         IBotanicalName n_abies_kawakamii = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1314         n_abies_kawakamii.setNameCache("Abies kawakamii", true);
1315         Taxon t_abies_kawakamii = Taxon.NewInstance(n_abies_kawakamii, sec);
1316         t_abies_kawakamii.getTitleCache();
1317         taxonService.save(t_abies_kawakamii);
1318
1319         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1320         Taxon t_abies_kawakamii_sensu_komarov = Taxon.NewInstance(n_abies_kawakamii, sec_sensu);
1321         taxonService.save(t_abies_kawakamii_sensu_komarov);
1322         t_abies_kawakamii_sensu_komarov.addTaxonRelation(t_abies_balsamea, TaxonRelationshipType.MISAPPLIED_NAME_FOR(), null, null);
1323         taxonService.saveOrUpdate(t_abies_kawakamii_sensu_komarov);
1324
1325         IBotanicalName n_abies_lasiocarpa = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1326         n_abies_lasiocarpa.setNameCache("Abies lasiocarpa", true);
1327         Taxon t_abies_lasiocarpa = Taxon.NewInstance(n_abies_lasiocarpa, sec);
1328         taxonService.save(t_abies_lasiocarpa);
1329
1330         // add taxa to classifications
1331         europeanAbiesClassification.addChildTaxon(t_abies_balsamea, null, null);
1332         alternativeClassification.addChildTaxon(t_abies_lasiocarpa, null, null);
1333         classificationService.saveOrUpdate(europeanAbiesClassification);
1334         classificationService.saveOrUpdate(alternativeClassification);
1335
1336         //
1337         // Description
1338         //
1339         TaxonDescription d_abies_alba = TaxonDescription.NewInstance(t_abies_alba);
1340         TaxonDescription d_abies_balsamea = TaxonDescription.NewInstance(t_abies_balsamea);
1341
1342         d_abies_alba.setUuid(UUID.fromString(D_ABIES_ALBA_UUID));
1343         d_abies_balsamea.setUuid(UUID.fromString(D_ABIES_BALSAMEA_UUID));
1344
1345
1346         // CommonTaxonName
1347         d_abies_alba.addElement(CommonTaxonName.NewInstance("Wei"+UTF8.SHARP_S+"tanne", Language.GERMAN()));
1348         d_abies_alba.addElement(CommonTaxonName.NewInstance("silver fir", Language.ENGLISH()));
1349         d_abies_alba.addElement(Distribution
1350                 .NewInstance(
1351                         germany,
1352                         PresenceAbsenceTerm.NATIVE()));
1353         d_abies_alba.addElement(Distribution
1354                 .NewInstance(
1355                         russia,
1356                         PresenceAbsenceTerm.ABSENT()));
1357
1358         // TextData
1359         d_abies_balsamea
1360             .addElement(TextData
1361                     .NewInstance(
1362                             "Die Balsam-Tanne (Abies balsamea) ist eine Pflanzenart aus der Gattung der Tannen (Abies). Sie wÃ¤chst im nordÃ¶stlichen Nordamerika, wo sie sowohl Tief- als auch Bergland besiedelt. Sie gilt als relativ anspruchslos gegenÃ¼ber dem Standort und ist frosthart. In vielen Teilen des natÃ¼rlichen Verbreitungsgebietes stellt sie die Klimaxbaumart dar.",
1363                             Language.GERMAN(), null));
1364         d_abies_balsamea
1365         .addElement(CommonTaxonName
1366                 .NewInstance(
1367                         "Balsam-Tanne",
1368                         Language.GERMAN(), null));
1369
1370         d_abies_balsamea
1371         .addElement(Distribution
1372                 .NewInstance(
1373                         canada,
1374                         PresenceAbsenceTerm.PRESENT()));
1375
1376         d_abies_balsamea
1377         .addElement(Distribution
1378                 .NewInstance(
1379                         germany,
1380                         PresenceAbsenceTerm.NATIVE()));
1381
1382         d_abies_balsamea
1383                 .addElement(TextData
1384                         .NewInstance(
1385                                 TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_LONG,
1386                                 Language.RUSSIAN(), null));
1387         d_abies_balsamea
1388         .addElement(CommonTaxonName
1389                 .NewInstance(
1390                         TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_SHORT,
1391                         Language.RUSSIAN(), null));
1392         descriptionService.saveOrUpdate(d_abies_balsamea);
1393
1394         setComplete();
1395         endTransaction();
1396
1397
1398         writeDbUnitDataSetFile(new String[] {
1399             "TAXONBASE", "TAXONNAME",
1400             "TAXONRELATIONSHIP",
1401             "REFERENCE", "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
1402             "AGENTBASE", "HOMOTYPICALGROUP",
1403             "CLASSIFICATION", "TAXONNODE",
1404             "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING",
1405             "HIBERNATE_SEQUENCES" // IMPORTANT!!!
1406             });
1407
1408     }
1409
1410     /**
1411      *
1412      */
1413     private void refreshLuceneIndex() {
1414
1415 //        commitAndStartNewTransaction(null);
1416         commit();
1417         endTransaction();
1418         indexer.purge(DefaultProgressMonitor.NewInstance());
1419         indexer.reindex(typesToIndex, DefaultProgressMonitor.NewInstance());
1420         startNewTransaction();
1421 //        commitAndStartNewTransaction(null);
1422     }
1423
1424     /**
1425      * @param numberOfNew
1426      *
1427      */
1428     private void createRandomTaxonWithCommonName(int numberOfNew) {
1429
1430         logger.debug(String.format("creating %1$s random taxan with CommonName", numberOfNew));
1431
1432         commitAndStartNewTransaction(null);
1433
1434         Reference sec = ReferenceFactory.newBook();
1435         referenceService.save(sec);
1436
1437         for (int i = numberOfNew; i < numberOfNew; i++) {
1438             RandomStringUtils.randomAlphabetic(10);
1439             String radomName = RandomStringUtils.randomAlphabetic(5) + " " + RandomStringUtils.randomAlphabetic(10);
1440             String radomCommonName = RandomStringUtils.randomAlphabetic(10);
1441
1442             IBotanicalName name = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1443             name.setNameCache(radomName, true);
1444             Taxon taxon = Taxon.NewInstance(name, sec);
1445             taxonService.save(taxon);
1446
1447             TaxonDescription description = TaxonDescription.NewInstance(taxon);
1448             description.addElement(CommonTaxonName.NewInstance(radomCommonName, Language.GERMAN()));
1449             descriptionService.save(description);
1450         }
1451
1452         commitAndStartNewTransaction(null);
1453     }
1454
1455     private <T extends CdmBase> void logFreeTextSearchResults(Pager<SearchResult<T>> pager, Level level, String[] docFields){
1456         if(level == null){
1457             level = Level.DEBUG;
1458         }
1459         if(logger.isEnabledFor(level)){
1460             StringBuilder b = new StringBuilder();
1461             b.append("\n");
1462             int i = 0;
1463             for(SearchResult sr : pager.getRecords()){
1464
1465                 b.append(" ").append(i++).append(" - ");
1466                 b.append("score:").append(sr.getScore()).append(", ");
1467
1468                 if(docFields != null){
1469                     b.append("docs : ");
1470                     for(Document doc : sr.getDocs()) {
1471                         b.append("<");
1472                         for(String f : docFields){
1473                             b.append(f).append(":").append(Arrays.toString(doc.getValues(f)));
1474                         }
1475                         b.append(">");
1476                     }
1477                 }
1478
1479                 CdmBase entity = sr.getEntity();
1480                 if(entity == null){
1481                     b.append("NULL");
1482                 } else {
1483                     b.append(entity.getClass().getSimpleName()).
1484                         append(" [").append(entity.getId()).
1485                         append(" | ").append(entity.getUuid()).append("] : ").
1486                         append(entity.toString());
1487
1488                 }
1489                 b.append("\n");
1490             }
1491             logger.log(level, b);
1492         }
1493     }
1494
1495 }