cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/TaxonServiceSearchTest.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import static org.junit.Assert.assertNotNull;
  13
  14 import java.io.FileNotFoundException;
  15 import java.io.IOException;
  16 import java.util.ArrayList;
  17 import java.util.Arrays;
  18 import java.util.EnumSet;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.RandomStringUtils;
  26 import org.apache.log4j.Level;
  27 import org.apache.log4j.Logger;
  28 import org.apache.lucene.document.Document;
  29 import org.junit.Assert;
  30 import org.junit.Before;
  31 import org.junit.Ignore;
  32 import org.junit.Test;
  33 import org.unitils.dbunit.annotation.DataSet;
  34 import org.unitils.spring.annotation.SpringBeanByType;
  35
  36 import eu.etaxonomy.cdm.api.service.config.FindTaxaAndNamesConfiguratorImpl;
  37 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  38 import eu.etaxonomy.cdm.api.service.pager.Pager;
  39 import eu.etaxonomy.cdm.api.service.search.ICdmMassIndexer;
  40 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  41 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  42 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  43 import eu.etaxonomy.cdm.common.UTF8;
  44 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  45 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  46 import eu.etaxonomy.cdm.model.common.CdmBase;
  47 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  48 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  49 import eu.etaxonomy.cdm.model.common.Language;
  50 import eu.etaxonomy.cdm.model.description.CategoricalData;
  51 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  52 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  53 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  54 import eu.etaxonomy.cdm.model.description.Distribution;
  55 import eu.etaxonomy.cdm.model.description.Feature;
  56 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  57 import eu.etaxonomy.cdm.model.description.State;
  58 import eu.etaxonomy.cdm.model.description.StateData;
  59 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  60 import eu.etaxonomy.cdm.model.description.TextData;
  61 import eu.etaxonomy.cdm.model.location.Country;
  62 import eu.etaxonomy.cdm.model.location.NamedArea;
  63 import eu.etaxonomy.cdm.model.name.IBotanicalName;
  64 import eu.etaxonomy.cdm.model.name.Rank;
  65 import eu.etaxonomy.cdm.model.name.TaxonName;
  66 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  67 import eu.etaxonomy.cdm.model.reference.Reference;
  68 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  69 import eu.etaxonomy.cdm.model.taxon.Classification;
  70 import eu.etaxonomy.cdm.model.taxon.Synonym;
  71 import eu.etaxonomy.cdm.model.taxon.SynonymType;
  72 import eu.etaxonomy.cdm.model.taxon.Taxon;
  73 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  74 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  75 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  76 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  77 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  78 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  79 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  80 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  81
  82 /**
  83  * @author a.babadshanjan, a.kohlbecker
  84  * @since 04.02.2009
  85  */
  86 public class TaxonServiceSearchTest extends CdmTransactionalIntegrationTest {
  87
  88     private static Logger logger = Logger.getLogger(TaxonServiceSearchTest.class);
  89
  90     private static final UUID ABIES_BALSAMEA_UUID = UUID.fromString("f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8");
  91     private static final UUID ABIES_ALBA_UUID = UUID.fromString("7dbd5810-a3e5-44b6-b563-25152b8867f4");
  92     private static final UUID CLASSIFICATION_UUID = UUID.fromString("2a5ceebb-4830-4524-b330-78461bf8cb6b");
  93     private static final UUID CLASSIFICATION_ALT_UUID = UUID.fromString("d7c741e3-ae9e-4a7d-a566-9e3a7a0b51ce");
  94     private static final UUID D_ABIES_BALSAMEA_UUID = UUID.fromString("900108d8-e6ce-495e-b32e-7aad3099135e");
  95     private static final UUID D_ABIES_ALBA_UUID = UUID.fromString("ec8bba03-d993-4c85-8472-18b14942464b");
  96     private static final UUID D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID = UUID.fromString("e9d8c2fd-6409-46d5-9c2e-14a2bbb1b2b1");
  97     private static final UUID ABIES_SUBALPINA_UUID = UUID.fromString("9fee273c-c819-4f1f-913a-cd910465df51");
  98
  99     private static final int NUM_OF_NEW_RADOM_ENTITIES = 1000;
 100
 101     private boolean includeUnpublished = true;
 102
 103
 104
 105     @SpringBeanByType
 106     private ITaxonService taxonService;
 107     @SpringBeanByType
 108     private ITermService termService;
 109     @SpringBeanByType
 110     private IClassificationService classificationService;
 111     @SpringBeanByType
 112     private IReferenceService referenceService;
 113     @SpringBeanByType
 114     private IDescriptionService descriptionService;
 115     @SpringBeanByType
 116     private INameService nameService;
 117     @SpringBeanByType
 118     private ICdmMassIndexer indexer;
 119
 120     @SpringBeanByType
 121     private ITaxonNodeService nodeService;
 122
 123     private static final int BENCHMARK_ROUNDS = 300;
 124
 125     private Set<Class<? extends CdmBase>> typesToIndex = null;
 126
 127     private NamedArea germany;
 128     private NamedArea france ;
 129     private NamedArea russia ;
 130     private NamedArea canada ;
 131
 132     @Before
 133     public void setUp() throws Exception {
 134         typesToIndex = new HashSet<>();
 135         typesToIndex.add(DescriptionElementBase.class);
 136         typesToIndex.add(TaxonBase.class);
 137         typesToIndex.add(TaxonRelationship.class);
 138
 139         germany =  Country.GERMANY();
 140         france = Country.FRANCEFRENCHREPUBLIC();
 141         russia = Country.RUSSIANFEDERATION();
 142         canada = Country.CANADA();
 143
 144         includeUnpublished = true;
 145     }
 146
 147     @Test
 148     public void testDbUnitUsageTest() throws Exception {
 149         assertNotNull("taxonService should exist", taxonService);
 150         assertNotNull("nameService should exist", nameService);
 151     }
 152
 153
 154     @SuppressWarnings("rawtypes")
 155     @Test
 156     @DataSet
 157     public final void testPurgeAndReindex() throws IOException, LuceneParseException {
 158
 159         refreshLuceneIndex();
 160
 161         Pager<SearchResult<TaxonBase>> pager = taxonService.findByFullText(null, "Abies", null, includeUnpublished,
 162                 null, true, null, null, null, null); // --> 8
 163         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 164
 165         indexer.purge(null);
 166         commitAndStartNewTransaction(null);
 167
 168         pager = taxonService.findByFullText(null, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 0
 169         Assert.assertEquals("Expecting no entities since the index has been purged", 0, pager.getCount().intValue());
 170
 171         indexer.reindex(indexer.indexedClasses(), null);
 172         commitAndStartNewTransaction(null);
 173
 174         pager = taxonService.findByFullText(null, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 8
 175         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 176     }
 177
 178
 179     @SuppressWarnings("rawtypes")
 180     @Test
 181     @DataSet
 182     public final void testFindByDescriptionElementFullText_CommonName() throws IOException,
 183             LuceneParseException {
 184
 185         refreshLuceneIndex();
 186
 187         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null, null,
 188                 false, null, null, null, null);
 189         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1,
 190                 pager.getCount().intValue());
 191
 192         // the description containing the Nulltanne has no taxon attached,
 193         // taxon.id = null
 194         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Nulltanne", null, null, null,
 195                 false, null, null, null, null);
 196         Assert.assertEquals("Expecting no entity when searching for 'Nulltanne' ", 0, pager.getCount().intValue());
 197
 198         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 199                 Arrays.asList(new Language[] { Language.GERMAN() }), false, null, null, null, null);
 200         Assert.assertEquals("Expecting one entity when searching in German", 1, pager.getCount().intValue());
 201
 202         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 203                 Arrays.asList(new Language[] { Language.RUSSIAN() }), false, null, null, null, null);
 204         Assert.assertEquals("Expecting no entity when searching in Russian", 0, pager.getCount().intValue());
 205
 206     }
 207
 208     @SuppressWarnings("rawtypes")
 209     @Test
 210     @DataSet
 211     public final void testFindByDescriptionElementFullText_Distribution() throws IOException, LuceneParseException {
 212
 213         refreshLuceneIndex();
 214
 215         // by Area
 216         Pager<SearchResult<TaxonBase>>  pager = taxonService.findByDescriptionElementFullText(null, "Canada", null, null, null, false, null, null, null, null);
 217         Assert.assertEquals("Expecting one entity when searching for arae 'Canada'", 1, pager.getCount().intValue());
 218         // by Status
 219         pager = taxonService.findByDescriptionElementFullText(null, "present", null, null, null, false, null, null, null, null);
 220         Assert.assertEquals("Expecting one entity when searching for status 'present'", 1, pager.getCount().intValue());
 221     }
 222
 223     @SuppressWarnings("rawtypes")
 224     @Test
 225     @DataSet
 226     public final void testFindByDescriptionElementFullText_wildcard() throws IOException, LuceneParseException {
 227
 228         refreshLuceneIndex();
 229
 230         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
 231         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1, pager.getCount().intValue());
 232     }
 233
 234     /**
 235      * Regression test for #3113 (hibernate search: wildcard query can cause BooleanQuery$TooManyClauses: maxClauseCount is set to 1024)
 236      *
 237      * @throws IOException
 238      * @throws LuceneParseException
 239      */
 240     @SuppressWarnings("rawtypes")
 241     @Test
 242     @DataSet
 243     public final void testFindByDescriptionElementFullText_TooManyClauses() throws IOException, LuceneParseException {
 244
 245         // generate 1024 terms to reproduce the bug
 246         TaxonDescription description = (TaxonDescription) descriptionService.find(D_ABIES_ALBA_UUID);
 247         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 248         while(uniqueRandomStrs.size() < 1024){
 249             uniqueRandomStrs.add(RandomStringUtils.random(10, true, false));
 250         }
 251         for(String rndStr: uniqueRandomStrs){
 252             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 253         }
 254         descriptionService.saveOrUpdate(description);
 255         commitAndStartNewTransaction(null);
 256
 257         refreshLuceneIndex();
 258
 259         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, null, null, null, null);
 260         Assert.assertEquals("Expecting all 1024 entities grouped into one SearchResult item when searching for Rot*", 1, pager.getCount().intValue());
 261     }
 262
 263     /**
 264      * Regression test for #3116 (fulltext search: always only one page of results)
 265      *
 266      * @throws IOException
 267      * @throws LuceneParseException
 268      */
 269     @SuppressWarnings("rawtypes")
 270     @Test
 271     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 272     @Ignore
 273     public final void testFullText_Paging() throws IOException, LuceneParseException {
 274
 275         Reference sec = ReferenceFactory.newDatabase();
 276         referenceService.save(sec);
 277
 278         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 279         int numOfItems = 100;
 280         while(uniqueRandomStrs.size() < numOfItems){
 281             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 282         }
 283
 284         for(String rndStr: uniqueRandomStrs){
 285
 286             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(Rank.SERIES()), sec);
 287             taxon.setTitleCache("Tax" + rndStr, true);
 288             taxonService.save(taxon);
 289
 290             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 291             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 292             descriptionService.saveOrUpdate(description);
 293         }
 294
 295         commitAndStartNewTransaction(new String[]{"TAXONBASE"});
 296         refreshLuceneIndex();
 297
 298         int pageSize = 10;
 299
 300         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, null, null, null);
 301         Assert.assertEquals("unexpeted number of pages", Integer.valueOf(numOfItems / pageSize), pager.getPagesAvailable());
 302         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 9, null, null);
 303         Assert.assertNotNull("last page must have records", pager.getRecords());
 304         Assert.assertNotNull("last item on last page must exist", pager.getRecords().get(0));
 305         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 10, null, null);
 306         Assert.assertNotNull("last page + 1 must not have any records", pager.getRecords());
 307     }
 308
 309     /**
 310      * test for max score and sort by score of hit groups
 311      * with all matches per taxon in a single TextData  element
 312      * see {@link #testFullText_ScoreAndOrder_2()} for the complement
 313      * test with matches in multiple TextData per taxon
 314      *
 315      * @throws IOException
 316      * @throws LuceneParseException
 317      */
 318     @SuppressWarnings("rawtypes")
 319     @Test
 320     @DataSet
 321     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 322     public final void testFullText_ScoreAndOrder_1() throws IOException, LuceneParseException {
 323
 324         int numOfTaxa = 3;
 325
 326         UUID[] taxonUuids = new UUID[numOfTaxa];
 327         StringBuilder text = new StringBuilder();
 328
 329         for(int i = 0; i < numOfTaxa; i++){
 330
 331             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 332             taxon.setTitleCache("Taxon_" + i, true);
 333             taxonUuids[i] = taxon.getUuid();
 334             taxonService.save(taxon);
 335
 336             text.append(" ").append("Rot");
 337             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 338             description.addElement(TextData.NewInstance(text.toString(), Language.DEFAULT(), null));
 339             descriptionService.saveOrUpdate(description);
 340         }
 341
 342         commitAndStartNewTransaction(null);
 343         refreshLuceneIndex();
 344
 345         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 346         for(int i = 0; i < numOfTaxa; i++){
 347             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 348         }
 349         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 350     }
 351
 352     /**
 353      * test for max score and sort by score of hit groups
 354      * with all matches per taxon in a multiple TextData elements
 355      * see {@link #testFullText_ScoreAndOrder_1()} for the complement
 356      * test with matches in a single TextData per taxon
 357      *
 358      * @throws IOException
 359      * @throws LuceneParseException
 360      */
 361     @SuppressWarnings("rawtypes")
 362     @Test
 363     @DataSet
 364     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 365     public final void testFullText_ScoreAndOrder_2() throws IOException, LuceneParseException {
 366
 367         int numOfTaxa = 3;
 368
 369         UUID[] taxonUuids = new UUID[numOfTaxa];
 370
 371         for(int i = 0; i < numOfTaxa; i++){
 372
 373             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 374             taxon.setTitleCache("Taxon_" + i, true);
 375             taxonUuids[i] = taxon.getUuid();
 376             taxonService.save(taxon);
 377
 378             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 379             for(int k = 0; k < i; k++){
 380                 description.addElement(TextData.NewInstance("Rot", Language.DEFAULT(), null));
 381             }
 382             descriptionService.saveOrUpdate(description);
 383         }
 384
 385         commitAndStartNewTransaction(null);
 386         refreshLuceneIndex();
 387
 388         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 389         for(int i = 0; i < numOfTaxa; i++){
 390             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 391         }
 392         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 393     }
 394
 395
 396     /**
 397      * @throws IOException
 398      * @throws LuceneParseException
 399      * @throws LuceneMultiSearchException
 400      */
 401     @Test
 402     @DataSet
 403     public final void testFullText_Grouping() throws IOException, LuceneParseException, LuceneMultiSearchException {
 404
 405         TaxonDescription description = (TaxonDescription) descriptionService.find(D_ABIES_ALBA_UUID);
 406         Set<String> uniqueRandomStrs = new HashSet<>(1024);
 407         int numOfItems = 100;
 408         while(uniqueRandomStrs.size() < numOfItems){
 409             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 410         }
 411         for(String rndStr: uniqueRandomStrs){
 412             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 413         }
 414         descriptionService.saveOrUpdate(description);
 415
 416         commitAndStartNewTransaction(new String[]{"DESCRIPTIONELEMENTBASE"});
 417
 418         refreshLuceneIndex();
 419
 420         int pageSize = 10;
 421
 422          boolean highlightFragments = true;
 423
 424         // test with findByDescriptionElementFullText
 425         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, highlightFragments, pageSize, null, null, null);
 426         logFreeTextSearchResults(pager, Level.DEBUG, null);
 427         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 428         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 429         Map<String, String[]> highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 430         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 431         int maxDocsPerGroup = 10;
 432         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 433
 434         // test with findByEverythingFullText
 435         pager = taxonService.findByEverythingFullText( "Rot*", null, includeUnpublished, null, highlightFragments, pageSize, null, null, null);
 436         logFreeTextSearchResults(pager, Level.DEBUG, null);
 437         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 438         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 439         highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 440         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 441         maxDocsPerGroup = 10;
 442         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 443
 444     }
 445
 446     @SuppressWarnings("rawtypes")
 447     @Test
 448     @DataSet
 449     @Ignore
 450     public final void testFindByDescriptionElementFullText_TextData() throws IOException, LuceneParseException {
 451
 452         refreshLuceneIndex();
 453
 454         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, false, null, null, null, null);
 455         logFreeTextSearchResults(pager, Level.DEBUG, null);
 456         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 457         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 458         Assert.assertTrue("Expecting two docs, one for RUSSIAN and one for GERMAN", pager.getRecords().get(0).getDocs().size() == 2);
 459         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 460
 461
 462         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, null, null, false, null, null, null, null);
 463         Assert.assertEquals("Expecting one entity when searching for any type", 1, pager.getCount().intValue());
 464
 465         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.UNKNOWN()}), null, false, null, null, null, null);
 466         Assert.assertEquals("Expecting one entity when searching for any type and for Feature DESCRIPTION", 1, pager.getCount().intValue());
 467
 468         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER()}), null, false, null, null, null, null);
 469         Assert.assertEquals("Expecting no entity when searching for any type and for Feature CHROMOSOME_NUMBER", 0, pager.getCount().intValue());
 470
 471         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER(), Feature.UNKNOWN()}), null, false, null, null, null, null);
 472         Assert.assertEquals("Expecting no entity when searching for any type and for Feature DESCRIPTION or CHROMOSOME_NUMBER", 1, pager.getCount().intValue());
 473
 474         pager = taxonService.findByDescriptionElementFullText(Distribution.class, "Abies", null, null, null, false, null, null, null, null);
 475         Assert.assertEquals("Expecting no entity when searching for Distribution", 0, pager.getCount().intValue());
 476
 477         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{}), false, null, null, null, null);
 478         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 479         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 480
 481         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.RUSSIAN()}), false, null, null, null, null);
 482         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 483         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 484
 485         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 486         Assert.assertEquals("Expecting no entity", 0, pager.getCount().intValue());
 487
 488         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 489         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 490         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 491     }
 492
 493     @SuppressWarnings("rawtypes")
 494     @Test
 495     @DataSet
 496     public final void testFindByDescriptionElementFullText_MultipleWords() throws IOException, LuceneParseException {
 497
 498         refreshLuceneIndex();
 499
 500         // Pflanzenart aus der Gattung der Tannen
 501         long start = System.currentTimeMillis();
 502
 503         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, false, null, null, null, null);
 504         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 505
 506         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Wespen", null, null, null, false, null, null, null, null);
 507         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 508
 509         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, false, null, null, null, null);
 510         Assert.assertEquals("AND search : Expecting one entity", 1, pager.getCount().intValue());
 511
 512         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Wespen", null, null, null, false, null, null, null, null);
 513         Assert.assertEquals("AND search : Expecting no entity", 0, pager.getCount().intValue());
 514
 515         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, false, null, null, null, null);
 516         Assert.assertEquals("Phrase search : Expecting one entity", 1, pager.getCount().intValue());
 517
 518         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Wespen\"", null, null, null, false, null, null, null, null);
 519         Assert.assertEquals("Phrase search : Expecting one entity", 0, pager.getCount().intValue());
 520
 521         logger.info("testFindByDescriptionElementFullText_MultipleWords() duration: " + (System.currentTimeMillis() - start) + "ms");
 522
 523     }
 524
 525
 526     @SuppressWarnings("rawtypes")
 527     @Test
 528     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 529     public final void testFindByDescriptionElementFullText_modify_DescriptionElement() throws IOException, LuceneParseException {
 530
 531         refreshLuceneIndex();
 532
 533         //
 534         // modify the DescriptionElement
 535         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 536         Assert.assertTrue("Search did not return any results", pager.getRecords().size() > 0);
 537         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 538         Document indexDocument = pager.getRecords().get(0).getDocs().iterator().next();
 539         String[] descriptionElementUuidStr = indexDocument.getValues("uuid");
 540         String[] inDescriptionUuidStr = indexDocument.getValues("inDescription.uuid");
 541         // is only one uuid!
 542         DescriptionElementBase textData = descriptionService.getDescriptionElementByUuid(UUID.fromString(descriptionElementUuidStr[0]));
 543
 544         ((TextData)textData).removeText(Language.GERMAN());
 545         ((TextData)textData).putText(Language.SPANISH_CASTILIAN(), "abeto bals"+UTF8.SMALL_A_ACUTE+"mico");
 546
 547         descriptionService.saveDescriptionElement(textData);
 548         commitAndStartNewTransaction(null);
 549 //        printDataSet(System.out, new String[] {
 550 //                "DESCRIPTIONELEMENTBASE", "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING" }
 551 //        );
 552
 553         //
 554         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 555         Assert.assertEquals("The german 'Balsam-Tanne' TextData should no longer be indexed", 0, pager.getCount().intValue());
 556         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 557         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 558         pager = taxonService.findByDescriptionElementFullText(TextData.class, "bals"+UTF8.SMALL_A_ACUTE+"mico", null, null, null, false, null, null, null, null);
 559         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 560
 561         //
 562         // modify the DescriptionElement via the Description object
 563         DescriptionBase<?> description = descriptionService.find(UUID.fromString(inDescriptionUuidStr[0]));
 564         Set<DescriptionElementBase> elements = description.getElements();
 565         for( DescriptionElementBase elm : elements){
 566             if(elm.getUuid().toString().equals(descriptionElementUuidStr[0])){
 567                 ((TextData)elm).removeText(Language.SPANISH_CASTILIAN());
 568                 ((TextData)elm).putText(Language.POLISH(), "Jod"+UTF8.POLISH_L+"a balsamiczna");
 569             }
 570         }
 571         descriptionService.saveOrUpdate(description);
 572         commitAndStartNewTransaction(null);
 573         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 574         Assert.assertEquals("The spanish 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico' TextData should no longer be indexed", 0, pager.getCount().intValue());
 575         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsamiczna", null, null, Arrays.asList(new Language[]{Language.POLISH()}), false, null, null, null, null);
 576         Assert.assertEquals("expecting to find the POLISH 'Jod"+UTF8.POLISH_L+"a balsamiczna'", 1, pager.getCount().intValue());
 577     }
 578
 579     @SuppressWarnings("rawtypes")
 580     @Test
 581     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 582     public final void testFindByDescriptionElementFullText_modify_Taxon() throws IOException, LuceneParseException {
 583
 584         refreshLuceneIndex();
 585
 586         Taxon t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
 587         TaxonDescription d_abies_balsamea = (TaxonDescription)descriptionService.find(D_ABIES_BALSAMEA_UUID);
 588
 589         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 590         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne'", 1, pager.getCount().intValue());
 591
 592         // exchange the Taxon with another one via the Taxon object
 593         // 1.) remove existing description:
 594         t_abies_balsamea.removeDescription(d_abies_balsamea);
 595
 596         taxonService.saveOrUpdate(t_abies_balsamea);
 597         commitAndStartNewTransaction(null);
 598
 599         t_abies_balsamea = (Taxon)taxonService.find(t_abies_balsamea.getUuid());
 600
 601         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 602         Assert.assertEquals("'Balsam-Tanne' should no longer be found", 0, pager.getCount().intValue());
 603
 604         // 2.) create new description and add to taxon:
 605         TaxonDescription d_abies_balsamea_new = TaxonDescription.NewInstance();
 606         d_abies_balsamea_new
 607                 .addElement(TextData
 608                         .NewInstance(
 609                                 "Die Balsamtanne ist mit bis zu 30 m Höhe ein mittelgro"+UTF8.SHARP_S+"er Baum und kann bis zu 200 Jahre alt werden",
 610                                 Language.GERMAN(), null));
 611         t_abies_balsamea.addDescription(d_abies_balsamea_new);
 612         // set authorshipCache to null to avoid validation exception,
 613         // this is maybe not needed in future,  see ticket #3344
 614         IBotanicalName abies_balsamea = CdmBase.deproxy(t_abies_balsamea.getName(), TaxonName.class);
 615         abies_balsamea.setAuthorshipCache(null);
 616         printDataSet(System.err, new String[] {"LANGUAGESTRING_AUD"});
 617         taxonService.saveOrUpdate(t_abies_balsamea);
 618         commitAndStartNewTransaction(null);
 619
 620 //        printDataSet(System.out, new String[] {
 621 //                "DESCRIPTIONBASE"
 622 //        });
 623
 624         pager = taxonService.findByDescriptionElementFullText(TextData.class, "mittelgro"+UTF8.SHARP_S+"er Baum", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 625         Assert.assertEquals("the taxon should be found via the new Description", 1, pager.getCount().intValue());
 626     }
 627
 628     @SuppressWarnings("rawtypes")
 629     @Test
 630     @DataSet
 631     public final void testFindByDescriptionElementFullText_modify_Classification() throws IOException, LuceneParseException {
 632
 633         refreshLuceneIndex();
 634
 635         // put taxon into other classification, new taxon node
 636         Classification classification = classificationService.find(CLASSIFICATION_UUID);
 637         Classification alternateClassification = classificationService.find(CLASSIFICATION_ALT_UUID);
 638
 639         // TODO: why is the test failing when the childNode is already retrieved here, and not after the following four lines?
 640         //TaxonNode childNode = classification.getChildNodes().iterator().next();
 641
 642         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 643         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne' even if filtering by classification", 1, pager.getCount().intValue());
 644         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 645         Assert.assertEquals("GERMAN 'Balsam-Tanne' should NOT be found in other classification", 0, pager.getCount().intValue());
 646
 647         // check for the right taxon node
 648         TaxonNode childNode = classification.getChildNodes().iterator().next();
 649         Assert.assertEquals("expecting Abies balsamea sec.", childNode.getTaxon().getUuid(), ABIES_BALSAMEA_UUID);
 650         Assert.assertEquals("expecting default classification", childNode.getClassification().getUuid(), CLASSIFICATION_UUID);
 651
 652         // moving the taxon around, the rootnode is only a proxy
 653         alternateClassification.setRootNode(HibernateProxyHelper.deproxy(alternateClassification.getRootNode(), TaxonNode.class));
 654         alternateClassification.addChildNode(childNode, null, null);
 655
 656         classificationService.saveOrUpdate(alternateClassification);
 657         commitAndStartNewTransaction(null);
 658
 659 //        printDataSet(System.out, new String[] {
 660 //            "TAXONBASE", "TAXONNODE", "CLASSIFICATION"
 661 //        });
 662
 663         // reload classification
 664         classification = classificationService.find(CLASSIFICATION_UUID);
 665         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 666         Assert.assertEquals("GERMAN 'Balsam-Tanne' should now be found in other classification", 1, pager.getCount().intValue());
 667
 668         classification.getChildNodes().clear();
 669         classificationService.saveOrUpdate(classification);
 670         commitAndStartNewTransaction(null);
 671
 672         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", classification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 673         Assert.assertEquals("Now the GERMAN 'Balsam-Tanne' should NOT be found in original classification", 0, pager.getCount().intValue());
 674
 675     }
 676
 677     @SuppressWarnings("rawtypes")
 678     @Test
 679     @DataSet
 680     public final void testFindByDescriptionElementFullText_CategoricalData() throws IOException, LuceneParseException {
 681
 682         // add CategoricalData
 683         DescriptionBase d_abies_balsamea = descriptionService.find(D_ABIES_BALSAMEA_UUID);
 684         // Categorical data
 685         CategoricalData cdata = CategoricalData.NewInstance();
 686         cdata.setFeature(Feature.DESCRIPTION());
 687         State state = State.NewInstance("green", "green", "gn");
 688
 689         StateData statedata = StateData.NewInstance(state);
 690         statedata.putModifyingText(Language.ENGLISH(), "always, even during winter");
 691         cdata.addStateData(statedata);
 692         d_abies_balsamea.addElement(cdata);
 693
 694         UUID termUUID = termService.save(state).getUuid();
 695         descriptionService.save(d_abies_balsamea);
 696
 697         commitAndStartNewTransaction(null);
 698
 699 //        printDataSet(System.out, new String[] {
 700 //                 "STATEDATA", "STATEDATA_DEFINEDTERMBASE", "STATEDATA_LANGUAGESTRING", "LANGUAGESTRING"});
 701
 702         refreshLuceneIndex();
 703
 704         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(CategoricalData.class, "green", null, null, null, false, null, null, null, null);
 705         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 706         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 707         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 708         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 709
 710
 711         //TODO modify the StateData
 712         TaxonBase taxon = pager.getRecords().get(0).getEntity();
 713
 714         String newName = "Quercus robur";
 715         taxon.setTitleCache(newName + " sec. ", true);
 716
 717         taxonService.saveOrUpdate(taxon);
 718         commitAndStartNewTransaction(null);
 719
 720         taxon = taxonService.find(taxon.getUuid());
 721         Assert.assertEquals(newName + " sec. ", taxon.getTitleCache());
 722         DefinedTermBase term = termService.find(termUUID);
 723
 724         termService.delete(term);
 725
 726     }
 727
 728     @SuppressWarnings("rawtypes")
 729     @Test
 730     @DataSet
 731     public final void testFindByDescriptionElementFullText_Highlighting() throws IOException, LuceneParseException {
 732
 733         refreshLuceneIndex();
 734
 735         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, true, null, null, null, null);
 736         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 737         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 738         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
 739         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 740         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Abies</B>"));
 741
 742         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, true, null, null, null, null);
 743         searchResult = pager.getRecords().get(0);
 744         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 745         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 746         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") || fragments[0].contains("<B>Tannen</B>"));
 747
 748         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, true, null, null, null, null);
 749         searchResult = pager.getRecords().get(0);
 750         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 751         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 752         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") && fragments[0].contains("<B>Tannen</B>"));
 753
 754         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, true, null, null, null, null);
 755         searchResult = pager.getRecords().get(0);
 756         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 757         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 758         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B> <B>aus</B> <B>der</B> <B>Gattung</B> <B>der</B> <B>Tannen</B>"));
 759
 760         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Gatt*", null, null, null, true, null, null, null, null);
 761         searchResult = pager.getRecords().get(0);
 762         Assert.assertTrue("Wildcard search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 763         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 764         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Gatt"));
 765     }
 766
 767
 768     @Test
 769     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 770     public final void testFindByFullText() throws IOException, LuceneParseException {
 771
 772         refreshLuceneIndex();
 773
 774         classificationService.find(CLASSIFICATION_UUID);
 775
 776         boolean NO_UNPUBLISHED = false;
 777
 778         Pager<SearchResult<TaxonBase>> pager = taxonService.findByFullText(null, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 7
 779 //        logFreeTextSearchResults(pager, Level.DEBUG, null);
 780         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 781
 782
 783         pager = taxonService.findByFullText(null, "Abies", null, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 784 //        logFreeTextSearchResults(pager, Level.DEBUG, null);
 785         Assert.assertEquals("Expecting 6 entities", 6, pager.getCount().intValue());
 786         Synonym abiesSubalpina = (Synonym)taxonService.find(ABIES_SUBALPINA_UUID);
 787
 788         //accepted published, syn not published
 789         abiesSubalpina.getAcceptedTaxon().setPublish(true);
 790         commitAndStartNewTransaction();
 791         pager = taxonService.findByFullText(null, "Abies", null, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 792         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 793
 794         //accepted published, syn published
 795         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 796         abiesSubalpina.setPublish(true);
 797         commitAndStartNewTransaction();
 798         pager = taxonService.findByFullText(null, "Abies", null, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 799         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 800
 801         //accepted not published, syn published
 802         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 803         abiesSubalpina.getAcceptedTaxon().setPublish(false);
 804         commitAndStartNewTransaction();
 805         pager = taxonService.findByFullText(null, "Abies", null, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 7
 806         Assert.assertEquals("Expecting 6 entities. Synonym and accepted should not be found, though synonym is published",
 807                 6, pager.getCount().intValue());
 808
 809         pager = taxonService.findByFullText(Taxon.class, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 6
 810         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 811
 812         pager = taxonService.findByFullText(Synonym.class, "Abies", null, includeUnpublished, null, true, null, null, null, null); // --> 1
 813         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 814         pager = taxonService.findByFullText(Synonym.class, "Abies", null, NO_UNPUBLISHED, null, true, null, null, null, null); // --> 1
 815         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 816
 817         pager = taxonService.findByFullText(TaxonBase.class, "sec", null, includeUnpublished, null, true, null, null, null, null); // --> 7
 818         Assert.assertEquals("Expecting 8 entities", 9, pager.getCount().intValue());
 819
 820         pager = taxonService.findByFullText(null, "genus", null, includeUnpublished, null, true, null, null, null, null); // --> 1
 821         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 822
 823         pager = taxonService.findByFullText(Taxon.class, "subalpina", null, includeUnpublished, null, true, null, null, null, null); // --> 0
 824         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 825
 826         // synonym in classification ???
 827     }
 828
 829     @Test
 830     @DataSet
 831     public final void testPrepareByAreaSearch() throws IOException, LuceneParseException {
 832
 833         List<PresenceAbsenceTerm> statusFilter = new ArrayList<>();
 834         List<NamedArea> areaFilter = new ArrayList<>();
 835         areaFilter.add(germany);
 836         areaFilter.add(canada);
 837         areaFilter.add(russia);
 838
 839         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDistribution(areaFilter, statusFilter, null, 20, 0, null, null);
 840         Assert.assertEquals("Expecting 2 entities", Integer.valueOf(2), Integer.valueOf(pager.getRecords().size()));
 841
 842     }
 843
 844     @Test
 845     @DataSet
 846     public final void testFindTaxaAndNamesByFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
 847
 848         refreshLuceneIndex();
 849
 850         Classification alternateClassification = classificationService.find(CLASSIFICATION_ALT_UUID);
 851         Synonym abiesSubalpina = (Synonym)taxonService.find(ABIES_SUBALPINA_UUID);
 852
 853         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
 854                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 855                 "Abies", null, null, null, null, true, null, null, null, null);
 856 //        logPagerRecords(pager, Level.DEBUG);
 857         Assert.assertEquals("doTaxa & doSynonyms & unpublished", 8, pager.getCount().intValue());
 858         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 859                 "Abies", null, null, null, null, true, null, null, null, null);
 860         Assert.assertEquals("doTaxa & doSynonyms, published only", 6, pager.getCount().intValue());
 861
 862         //accepted published, syn not published
 863         abiesSubalpina.getAcceptedTaxon().setPublish(true);
 864         commitAndStartNewTransaction();
 865         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 866                 "Abies", null, null, null, null, true, null, null, null, null);
 867         Assert.assertEquals("doTaxa & doSynonyms, accepted published", 7, pager.getCount().intValue());
 868
 869         //accepted published, syn published
 870         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 871         abiesSubalpina.setPublish(true);
 872         commitAndStartNewTransaction();
 873         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 874                 "Abies", null, null, null, null, true, null, null, null, null);
 875         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 876
 877         //accepted not published, syn published
 878         abiesSubalpina = (Synonym)taxonService.find(abiesSubalpina.getUuid());
 879         abiesSubalpina.getAcceptedTaxon().setPublish(false);
 880         commitAndStartNewTransaction();
 881         pager = taxonService.findTaxaAndNamesByFullText(TaxaAndNamesSearchMode.taxaAndSynonyms(),
 882                 "Abies", null, null, null, null, true, null, null, null, null);
 883         Assert.assertEquals("Expecting 6 entities. Synonym and accepted should not be found, though synonym is published",
 884                 6, pager.getCount().intValue());
 885
 886
 887         EnumSet<TaxaAndNamesSearchMode> searchMode = EnumSet.allOf(TaxaAndNamesSearchMode.class);
 888         pager = taxonService.findTaxaAndNamesByFullText(
 889                 searchMode, "Abies", null, null, null, null, true, null, null, null, null);
 890 //        logPagerRecords(pager, Level.DEBUG);
 891         Assert.assertEquals("all search modes", 8, pager.getCount().intValue());
 892         searchMode.remove(TaxaAndNamesSearchMode.includeUnpublished);
 893         pager = taxonService.findTaxaAndNamesByFullText(
 894                 searchMode, "Abies", null, null, null, null, true, null, null, null, null);
 895         Assert.assertEquals("all search modes except unpublished", 6, pager.getCount().intValue());
 896
 897         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.allOf(TaxaAndNamesSearchMode.class),
 898                 "Abies", alternateClassification, null, null, null, true, null, null, null, null);
 899 //        logPagerRecords(pager, Level.DEBUG);
 900         Assert.assertEquals("all search modes, filtered by alternateClassification", 1, pager.getCount().intValue());
 901
 902         pager = taxonService.findTaxaAndNamesByFullText(
 903                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 904                 "Abies", null, null, null, null, true, null, null, null, null);
 905         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 906         Assert.assertEquals(Synonym.class, pager.getRecords().get(0).getEntity().getClass());
 907         //without published
 908         pager = taxonService.findTaxaAndNamesByFullText(EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
 909                 "Abies", null, null, null, null, true, null, null, null, null);
 910         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 911
 912
 913         pager = taxonService.findTaxaAndNamesByFullText(
 914                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
 915                 "Abies", null, null, null, null, true, null, null, null, null);
 916         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 917
 918         pager = taxonService.findTaxaAndNamesByFullText(
 919                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
 920                 "Tanne", null, null, null, null, true, null, null, null, null);
 921         Assert.assertEquals("Expecting 1 entity", 1, pager.getRecords().size());
 922         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 923         pager = taxonService.findTaxaAndNamesByFullText(
 924                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 925                 "Tanne", null, null, null, null, true, null, null, null, null);
 926         Assert.assertEquals("Expecting 0 entity", 0, pager.getRecords().size());
 927
 928         //misapplied names
 929         pager = taxonService.findTaxaAndNamesByFullText(
 930                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
 931                 "kawakamii", (Classification)null, null, null, null, true, null, null, null, null);
 932         logFreeTextSearchResults(pager, Level.DEBUG, null);
 933         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 934         //unpublish accepted taxon
 935         pager = taxonService.findTaxaAndNamesByFullText(
 936                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
 937                 "kawakamii", (Classification)null, null, null, null, true, null, null, null, null);
 938         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 939         //published accepted taxon/misapplied name
 940         Taxon abiesBalsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
 941         abiesBalsamea.setPublish(true);
 942         commitAndStartNewTransaction();
 943         pager = taxonService.findTaxaAndNamesByFullText(
 944                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
 945                 "kawakamii", (Classification)null, null, null, null, true, null, null, null, null);
 946         Assert.assertEquals("Expecting 1 entities", 1, pager.getCount().intValue());
 947         //unpublished misapplied name
 948         Taxon misapplied = (Taxon)taxonService.find(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
 949         misapplied.setPublish(false);
 950         commitAndStartNewTransaction();
 951         pager = taxonService.findTaxaAndNamesByFullText(
 952                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
 953                 "kawakamii", (Classification)null, null, null, null, true, null, null, null, null);
 954         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 955
 956
 957     }
 958
 959     @Test
 960     @DataSet
 961     public final void testFindTaxaAndNamesByFullText_PhraseQuery() throws IOException, LuceneParseException, LuceneMultiSearchException {
 962
 963         refreshLuceneIndex();
 964
 965         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
 966                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 967                 "\"Abies alba\"", null, null, null, null, true, null, null, null, null);
 968 //        logPagerRecords(pager, Level.DEBUG);
 969         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 1, pager.getCount().intValue());
 970
 971         pager = taxonService.findTaxaAndNamesByFullText(
 972                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 973                 "\"Abies al*\"", null, null, null, null, true, null, null, null, null);
 974 //        logPagerRecords(pager, Level.DEBUG);
 975         Assert.assertEquals("doTaxa & doSynonyms with complex phrase query", 1, pager.getCount().intValue());
 976
 977         pager = taxonService.findTaxaAndNamesByFullText(
 978                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
 979                 "\"Abies*\"", null, null, null, null, true, null, null, null, null);
 980 //        logPagerRecords(pager, Level.DEBUG);
 981         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 8, pager.getCount().intValue());
 982
 983     }
 984
 985     @Test
 986     @DataSet
 987     public final void testFindTaxaAndNamesByFullText_Sort() throws IOException, LuceneParseException, LuceneMultiSearchException {
 988
 989         refreshLuceneIndex();
 990
 991         List<OrderHint> orderHints = new ArrayList<>();
 992
 993 //        String[] docFields2log = new String[]{"id"};
 994
 995         // SortById
 996         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
 997         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
 998                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
 999                 "Abies", null, null, null, null, true, null, null, orderHints, null);
1000 //        logSearchResults(pager, Level.DEBUG, docFields2log);
1001         int lastId = -1;
1002         for(SearchResult<TaxonBase> rs : pager.getRecords()){
1003             if(lastId != -1){
1004                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
1005             }
1006             lastId = rs.getEntity().getId();
1007         }
1008
1009         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
1010         pager = taxonService.findTaxaAndNamesByFullText(
1011                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1012                 "Abies", null, null, null, null, true, null, null, orderHints, null);
1013 //        logSearchResults(pager, Level.DEBUG, docFields2log);
1014
1015         lastId = -1;
1016         for(SearchResult<TaxonBase> rs : pager.getRecords()){
1017             if(lastId != -1){
1018                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
1019             }
1020             lastId = rs.getEntity().getId();
1021         }
1022
1023         // Sortby NOMENCLATURAL_SORT_ORDER TODO make assertions !!!
1024         orderHints.clear();
1025         orderHints.addAll(OrderHint.NOMENCLATURAL_SORT_ORDER.asList());
1026         pager = taxonService.findTaxaAndNamesByFullText(
1027                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1028                 "Abies", null, null, null, null, true, null, null, orderHints, null);
1029         logFreeTextSearchResults(pager, Level.DEBUG, null);
1030
1031     }
1032
1033     @Test
1034     @DataSet
1035     public final void testFindTaxaAndNamesByFullText_AreaFilter() throws IOException, LuceneParseException, LuceneMultiSearchException {
1036
1037         refreshLuceneIndex();
1038
1039         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1040         a_germany_canada_russia.add(germany);
1041         a_germany_canada_russia.add(canada);
1042         a_germany_canada_russia.add(russia);
1043
1044         Set<NamedArea> a_russia = new HashSet<>();
1045         a_russia.add(russia);
1046
1047         Set<PresenceAbsenceTerm> present = new HashSet<>();
1048         present.add(PresenceAbsenceTerm.PRESENT());
1049
1050         Set<PresenceAbsenceTerm> present_native = new HashSet<>();
1051         present_native.add(PresenceAbsenceTerm.PRESENT());
1052         present_native.add(PresenceAbsenceTerm.NATIVE());
1053
1054         Set<PresenceAbsenceTerm> absent = new HashSet<>();
1055         absent.add(PresenceAbsenceTerm.ABSENT());
1056
1057         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1058                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.includeUnpublished),
1059                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1060         logFreeTextSearchResults(pager, Level.DEBUG, null);
1061
1062         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1063         pager = taxonService.findTaxaAndNamesByFullText(
1064                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1065                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1066         Assert.assertEquals("synonyms with matching area filter", 1, pager.getCount().intValue());
1067
1068         pager = taxonService.findTaxaAndNamesByFullText(
1069                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1070                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1071         logFreeTextSearchResults(pager, Level.DEBUG, null);
1072         Assert.assertEquals("taxa and synonyms with matching area filter", 3, pager.getCount().intValue());
1073
1074         pager = taxonService.findTaxaAndNamesByFullText(
1075                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1076                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1077         Assert.assertEquals("taxa and synonyms with matching area & status filter 1", 3, pager.getCount().intValue());
1078
1079         pager = taxonService.findTaxaAndNamesByFullText(
1080                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1081                 "Abies", null, a_germany_canada_russia, present, null, true, null, null, null, null);
1082         Assert.assertEquals("taxa and synonyms with matching area & status filter 2", 2, pager.getCount().intValue());
1083
1084         pager = taxonService.findTaxaAndNamesByFullText(
1085                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms, TaxaAndNamesSearchMode.includeUnpublished),
1086                 "Abies", null, a_russia, present, null, true, null, null, null, null);
1087         Assert.assertEquals("taxa and synonyms with non matching area & status filter", 0, pager.getCount().intValue());
1088
1089         pager = taxonService.findTaxaAndNamesByFullText(
1090                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames, TaxaAndNamesSearchMode.includeUnpublished),
1091                 "Tanne", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1092         Assert.assertEquals("ByCommonNames with area filter", 1, pager.getCount().intValue());
1093
1094         // abies_kawakamii_sensu_komarov as misapplied name for t_abies_balsamea
1095         pager = taxonService.findTaxaAndNamesByFullText(
1096                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1097                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1098         Assert.assertEquals("misappliedNames with matching area & status filter", 1, pager.getCount().intValue());
1099
1100
1101         // 1. remove existing taxon relation
1102         Taxon t_abies_balsamea = (Taxon)taxonService.find(ABIES_BALSAMEA_UUID);
1103         Set<TaxonRelationship> relsTo = t_abies_balsamea.getRelationsToThisTaxon();
1104         Assert.assertEquals(1, relsTo.size());
1105         TaxonRelationship taxonRelation = relsTo.iterator().next();
1106         t_abies_balsamea.removeTaxonRelation(taxonRelation);
1107         taxonService.saveOrUpdate(t_abies_balsamea);
1108         commitAndStartNewTransaction(null);
1109
1110         pager = taxonService.findTaxaAndNamesByFullText(
1111                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1112                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1113         Assert.assertEquals("misappliedNames with matching area & status filter, should match nothing now", 0, pager.getCount().intValue());
1114
1115         // 2. now add abies_kawakamii_sensu_komarov as misapplied name for t_abies_alba and search for misapplications in russia: ABSENT
1116         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1117         Taxon t_abies_alba = (Taxon)taxonService.find(ABIES_ALBA_UUID);
1118         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1119
1120         taxonService.update(t_abies_kawakamii_sensu_komarov);
1121
1122         commitAndStartNewTransaction(null);
1123
1124         pager = taxonService.findTaxaAndNamesByFullText(
1125                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames, TaxaAndNamesSearchMode.includeUnpublished),
1126                 "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1127         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1128
1129     }
1130
1131     @Test
1132     @DataSet
1133     //http://dev.e-taxonomy.eu/trac/ticket/5477
1134     public final void testFindTaxaAndNamesByFullText_AreaFilter_issue5477() throws IOException, LuceneParseException, LuceneMultiSearchException {
1135
1136         Set<NamedArea> a_germany_canada_russia = new HashSet<>();
1137         a_germany_canada_russia.add(germany);
1138         a_germany_canada_russia.add(canada);
1139         a_germany_canada_russia.add(russia);
1140
1141
1142         Set<PresenceAbsenceTerm> absent = new HashSet<>();
1143         absent.add(PresenceAbsenceTerm.ABSENT());
1144
1145         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID);
1146         Taxon t_abies_alba = (Taxon)taxonService.find(ABIES_ALBA_UUID);
1147         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1148
1149         /* Since the upgrade from hibernate search 4 to 5.5
1150          * triggering an update of t_abies_alba is no longer sufficient to also update the
1151          * document of t_abies_kawakamii_sensu_komarov in the lucene index.
1152          * the last test in testFindTaxaAndNamesByFullText_AreaFilter() failed in this case.
1153          * This situation is reproduced here:
1154          */
1155         taxonService.update(t_abies_alba);
1156
1157         commitAndStartNewTransaction(null);
1158
1159         Pager<SearchResult<TaxonBase>> pager = taxonService.findTaxaAndNamesByFullText(
1160                   EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1161                   "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1162         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1163     }
1164
1165
1166     /**
1167      * Regression test for #3119: fulltext search: Entity always null whatever search
1168      *
1169      * @throws IOException
1170      * @throws LuceneParseException
1171      * @throws LuceneMultiSearchException
1172      */
1173     @Test
1174     @DataSet
1175     public final void testFindByEverythingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1176
1177         refreshLuceneIndex();
1178
1179         // via Taxon
1180         Pager<SearchResult<TaxonBase>>pager = taxonService.findByEverythingFullText("Abies", null, includeUnpublished, null, true, null, null, null, null);
1181         logFreeTextSearchResults(pager, Level.DEBUG, null);
1182         Assert.assertTrue("Expecting at least 7 entities for 'Abies'", pager.getCount() > 7);
1183         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1184         Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1185
1186         // via DescriptionElement
1187         pager = taxonService.findByEverythingFullText("present", null, includeUnpublished, null, true, null, null, null, null);
1188         Assert.assertEquals("Expecting one entity when searching for area 'present'", 1, pager.getCount().intValue());
1189         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1190         Assert.assertEquals("Expecting Taxon entity", Taxon.class, CdmBase.deproxy(pager.getRecords().get(0).getEntity()).getClass());
1191         Assert.assertEquals("Expecting Taxon ", ABIES_BALSAMEA_UUID, pager.getRecords().get(0).getEntity().getUuid());
1192
1193     }
1194
1195
1196     @Test
1197     @DataSet
1198     public final void findByEveryThingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1199
1200         refreshLuceneIndex();
1201
1202         Pager<SearchResult<TaxonBase>> pager = taxonService.findByEverythingFullText("genus", null, includeUnpublished, null, false, null, null, null, null); // --> 1
1203         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1204
1205         //FIXME FAILS: abies balamea is returned twice, see also testFullText_Grouping()
1206         pager = taxonService.findByEverythingFullText("Balsam", null, includeUnpublished, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1207         logFreeTextSearchResults(pager, Level.DEBUG, null);
1208         Assert.assertEquals("expecting to find the Abies balsamea via the GERMAN DescriptionElements", 1, pager.getCount().intValue());
1209
1210         pager = taxonService.findByEverythingFullText("Abies", null, includeUnpublished, null, true, null, null, null, null);
1211         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
1212         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
1213         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
1214         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
1215         Assert.assertTrue("first fragments should contains serch term", fragments[0].toLowerCase().contains("<b>abies</b>"));
1216     }
1217
1218 //    @SuppressWarnings("rawtypes")
1219 //    @Test
1220 //    @DataSet
1221 //    public final void benchmarkFindTaxaAndNamesHql() throws IOException, LuceneParseException {
1222 //
1223 //        createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1224 //
1225 //        IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1226 //        configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1227 //        configurator.setMatchMode(MatchMode.BEGINNING);
1228 //        configurator.setDoTaxa(false);
1229 //        configurator.setDoSynonyms(false);
1230 //        configurator.setDoNamesWithoutTaxa(false);
1231 //        configurator.setDoTaxaByCommonNames(true);
1232 //
1233 //        Pager<IdentifiableEntity> pager;
1234 //
1235 //        long startMillis = System.currentTimeMillis();
1236 //        for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1237 //            pager = taxonService.findTaxaAndNames(configurator);
1238 //            if (logger.isDebugEnabled()) {
1239 //                logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1240 //            }
1241 //        }
1242 //        double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1243 //        logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1244 //    }
1245
1246     @SuppressWarnings("rawtypes")
1247     @Test
1248     @DataSet
1249     public final void benchmarkFindByCommonNameHql() {
1250
1251 //        printDataSet(System.err, new String[] { "TaxonBase" });
1252
1253         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1254
1255         IFindTaxaAndNamesConfigurator configurator = FindTaxaAndNamesConfiguratorImpl.NewInstance();
1256         configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1257         configurator.setMatchMode(MatchMode.BEGINNING);
1258         configurator.setDoTaxa(false);
1259         configurator.setDoSynonyms(false);
1260         configurator.setDoNamesWithoutTaxa(false);
1261         configurator.setDoTaxaByCommonNames(true);
1262
1263         Pager<IdentifiableEntity> pager;
1264
1265         long startMillis = System.currentTimeMillis();
1266         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1267             pager = taxonService.findTaxaAndNames(configurator);
1268             if (logger.isDebugEnabled()) {
1269                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1270             }
1271         }
1272         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1273         logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1274     }
1275
1276     @SuppressWarnings("rawtypes")
1277     @Test
1278     @DataSet
1279     public final void benchmarkFindByCommonNameLucene() throws IOException, LuceneParseException {
1280
1281         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1282
1283         refreshLuceneIndex();
1284
1285         Pager<SearchResult<TaxonBase>> pager;
1286
1287         long startMillis = System.currentTimeMillis();
1288         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1289             pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
1290             if (logger.isDebugEnabled()) {
1291                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getEntity().getTitleCache());
1292             }
1293         }
1294         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1295         logger.info("Benchmark result - [find taxon by CommonName via lucene] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1296     }
1297
1298     /**
1299      * uncomment @Test annotation to create the dataset for this test
1300      */
1301     @Override
1302     //    @Test
1303     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="BlankDataSet.xml")
1304     public final void createTestDataSet() throws FileNotFoundException {
1305
1306         Classification europeanAbiesClassification = Classification.NewInstance("European Abies");
1307         europeanAbiesClassification.setUuid(CLASSIFICATION_UUID);
1308         classificationService.save(europeanAbiesClassification);
1309
1310         Classification alternativeClassification = Classification.NewInstance("Abies alternative");
1311         alternativeClassification.setUuid(CLASSIFICATION_ALT_UUID);
1312         classificationService.save(alternativeClassification);
1313
1314         Reference sec = ReferenceFactory.newBook();
1315         sec.setTitleCache("Kohlbecker, A., Testcase standart views, 2013", true);
1316         Reference sec_sensu = ReferenceFactory.newBook();
1317         sec_sensu.setTitleCache("Komarov, V. L., Flora SSSR 29", true);
1318         referenceService.save(sec);
1319         referenceService.save(sec_sensu);
1320
1321         IBotanicalName n_abies = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
1322         n_abies.setNameCache("Abies", true);
1323         Taxon t_abies = Taxon.NewInstance(n_abies, sec);
1324         taxonService.save(t_abies);
1325
1326         IBotanicalName n_abies_alba = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1327         n_abies_alba.setNameCache("Abies alba", true);
1328         Taxon t_abies_alba = Taxon.NewInstance(n_abies_alba, sec);
1329         t_abies_alba.setUuid(ABIES_ALBA_UUID);
1330         taxonService.save(t_abies_alba);
1331
1332         IBotanicalName n_abies_subalpina = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1333         n_abies_subalpina.setNameCache("Abies subalpina", true);
1334         Synonym s_abies_subalpina = Synonym.NewInstance(n_abies_subalpina, sec);
1335         taxonService.save(s_abies_subalpina);
1336
1337         IBotanicalName n_abies_balsamea = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1338         n_abies_balsamea.setNameCache("Abies balsamea", true);
1339         Taxon t_abies_balsamea = Taxon.NewInstance(n_abies_balsamea, sec);
1340         t_abies_balsamea.setUuid(ABIES_BALSAMEA_UUID);
1341         t_abies_balsamea.addSynonym(s_abies_subalpina, SynonymType.SYNONYM_OF());
1342         taxonService.save(t_abies_balsamea);
1343
1344         IBotanicalName n_abies_grandis = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1345         n_abies_grandis.setNameCache("Abies grandis", true);
1346         Taxon t_abies_grandis = Taxon.NewInstance(n_abies_grandis, sec);
1347         taxonService.save(t_abies_grandis);
1348
1349         IBotanicalName n_abies_kawakamii = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1350         n_abies_kawakamii.setNameCache("Abies kawakamii", true);
1351         Taxon t_abies_kawakamii = Taxon.NewInstance(n_abies_kawakamii, sec);
1352         t_abies_kawakamii.getTitleCache();
1353         taxonService.save(t_abies_kawakamii);
1354
1355         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1356         Taxon t_abies_kawakamii_sensu_komarov = Taxon.NewInstance(n_abies_kawakamii, sec_sensu);
1357         taxonService.save(t_abies_kawakamii_sensu_komarov);
1358         t_abies_kawakamii_sensu_komarov.addTaxonRelation(t_abies_balsamea, TaxonRelationshipType.MISAPPLIED_NAME_FOR(), null, null);
1359         taxonService.saveOrUpdate(t_abies_kawakamii_sensu_komarov);
1360
1361         IBotanicalName n_abies_lasiocarpa = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1362         n_abies_lasiocarpa.setNameCache("Abies lasiocarpa", true);
1363         Taxon t_abies_lasiocarpa = Taxon.NewInstance(n_abies_lasiocarpa, sec);
1364         taxonService.save(t_abies_lasiocarpa);
1365
1366         // add taxa to classifications
1367         europeanAbiesClassification.addChildTaxon(t_abies_balsamea, null, null);
1368         alternativeClassification.addChildTaxon(t_abies_lasiocarpa, null, null);
1369         classificationService.saveOrUpdate(europeanAbiesClassification);
1370         classificationService.saveOrUpdate(alternativeClassification);
1371
1372         //
1373         // Description
1374         //
1375         TaxonDescription d_abies_alba = TaxonDescription.NewInstance(t_abies_alba);
1376         TaxonDescription d_abies_balsamea = TaxonDescription.NewInstance(t_abies_balsamea);
1377
1378         d_abies_alba.setUuid(D_ABIES_ALBA_UUID);
1379         d_abies_balsamea.setUuid(D_ABIES_BALSAMEA_UUID);
1380
1381
1382         // CommonTaxonName
1383         d_abies_alba.addElement(CommonTaxonName.NewInstance("Wei"+UTF8.SHARP_S+"tanne", Language.GERMAN()));
1384         d_abies_alba.addElement(CommonTaxonName.NewInstance("silver fir", Language.ENGLISH()));
1385         d_abies_alba.addElement(Distribution
1386                 .NewInstance(
1387                         germany,
1388                         PresenceAbsenceTerm.NATIVE()));
1389         d_abies_alba.addElement(Distribution
1390                 .NewInstance(
1391                         russia,
1392                         PresenceAbsenceTerm.ABSENT()));
1393
1394         // TextData
1395         d_abies_balsamea
1396             .addElement(TextData
1397                     .NewInstance(
1398                             "Die Balsam-Tanne (Abies balsamea) ist eine Pflanzenart aus der Gattung der Tannen (Abies). Sie wÃ¤chst im nordÃ¶stlichen Nordamerika, wo sie sowohl Tief- als auch Bergland besiedelt. Sie gilt als relativ anspruchslos gegenÃ¼ber dem Standort und ist frosthart. In vielen Teilen des natÃ¼rlichen Verbreitungsgebietes stellt sie die Klimaxbaumart dar.",
1399                             Language.GERMAN(), null));
1400         d_abies_balsamea
1401         .addElement(CommonTaxonName
1402                 .NewInstance(
1403                         "Balsam-Tanne",
1404                         Language.GERMAN(), null));
1405
1406         d_abies_balsamea
1407         .addElement(Distribution
1408                 .NewInstance(
1409                         canada,
1410                         PresenceAbsenceTerm.PRESENT()));
1411
1412         d_abies_balsamea
1413         .addElement(Distribution
1414                 .NewInstance(
1415                         germany,
1416                         PresenceAbsenceTerm.NATIVE()));
1417
1418         d_abies_balsamea
1419                 .addElement(TextData
1420                         .NewInstance(
1421                                 TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_LONG,
1422                                 Language.RUSSIAN(), null));
1423         d_abies_balsamea
1424         .addElement(CommonTaxonName
1425                 .NewInstance(
1426                         TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_SHORT,
1427                         Language.RUSSIAN(), null));
1428         descriptionService.saveOrUpdate(d_abies_balsamea);
1429
1430         setComplete();
1431         endTransaction();
1432
1433
1434         writeDbUnitDataSetFile(new String[] {
1435             "TAXONBASE", "TAXONNAME",
1436             "TAXONRELATIONSHIP",
1437             "REFERENCE", "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
1438             "AGENTBASE", "HOMOTYPICALGROUP",
1439             "CLASSIFICATION", "TAXONNODE",
1440             "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING",
1441             "HIBERNATE_SEQUENCES" // IMPORTANT!!!
1442             });
1443
1444     }
1445
1446     /**
1447      *
1448      */
1449     private void refreshLuceneIndex() {
1450
1451 //        commitAndStartNewTransaction(null);
1452         commit();
1453         endTransaction();
1454         indexer.purge(DefaultProgressMonitor.NewInstance());
1455         indexer.reindex(typesToIndex, DefaultProgressMonitor.NewInstance());
1456         startNewTransaction();
1457 //        commitAndStartNewTransaction(null);
1458     }
1459
1460     /**
1461      * @param numberOfNew
1462      *
1463      */
1464     private void createRandomTaxonWithCommonName(int numberOfNew) {
1465
1466         logger.debug(String.format("creating %1$s random taxan with CommonName", numberOfNew));
1467
1468         commitAndStartNewTransaction(null);
1469
1470         Reference sec = ReferenceFactory.newBook();
1471         referenceService.save(sec);
1472
1473         for (int i = numberOfNew; i < numberOfNew; i++) {
1474             RandomStringUtils.randomAlphabetic(10);
1475             String radomName = RandomStringUtils.randomAlphabetic(5) + " " + RandomStringUtils.randomAlphabetic(10);
1476             String radomCommonName = RandomStringUtils.randomAlphabetic(10);
1477
1478             IBotanicalName name = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1479             name.setNameCache(radomName, true);
1480             Taxon taxon = Taxon.NewInstance(name, sec);
1481             taxonService.save(taxon);
1482
1483             TaxonDescription description = TaxonDescription.NewInstance(taxon);
1484             description.addElement(CommonTaxonName.NewInstance(radomCommonName, Language.GERMAN()));
1485             descriptionService.save(description);
1486         }
1487
1488         commitAndStartNewTransaction(null);
1489     }
1490
1491     private <T extends CdmBase> void logFreeTextSearchResults(Pager<SearchResult<T>> pager, Level level, String[] docFields){
1492         if(level == null){
1493             level = Level.DEBUG;
1494         }
1495         if(logger.isEnabledFor(level)){
1496             StringBuilder b = new StringBuilder();
1497             b.append("\n");
1498             int i = 0;
1499             for(SearchResult<?> sr : pager.getRecords()){
1500
1501                 b.append(" ").append(i++).append(" - ");
1502                 b.append("score:").append(sr.getScore()).append(", ");
1503
1504                 if(docFields != null){
1505                     b.append("docs : ");
1506                     for(Document doc : sr.getDocs()) {
1507                         b.append("<");
1508                         for(String f : docFields){
1509                             b.append(f).append(":").append(Arrays.toString(doc.getValues(f)));
1510                         }
1511                         b.append(">");
1512                     }
1513                 }
1514
1515                 CdmBase entity = sr.getEntity();
1516                 if(entity == null){
1517                     b.append("NULL");
1518                 } else {
1519                     b.append(entity.getClass().getSimpleName()).
1520                         append(" [").append(entity.getId()).
1521                         append(" | ").append(entity.getUuid()).append("] : ").
1522                         append(entity.toString());
1523
1524                 }
1525                 b.append("\n");
1526             }
1527             logger.log(level, b);
1528         }
1529     }
1530
1531 }