cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/TaxonServiceSearchTest.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import static org.junit.Assert.assertNotNull;
  13
  14 import java.io.FileNotFoundException;
  15 import java.io.IOException;
  16 import java.util.ArrayList;
  17 import java.util.Arrays;
  18 import java.util.EnumSet;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.RandomStringUtils;
  26 import org.apache.log4j.Level;
  27 import org.apache.log4j.Logger;
  28 import org.apache.lucene.document.Document;
  29 import org.junit.Assert;
  30 import org.junit.Before;
  31 import org.junit.Ignore;
  32 import org.junit.Test;
  33 import org.unitils.dbunit.annotation.DataSet;
  34 import org.unitils.spring.annotation.SpringBeanByType;
  35
  36 import eu.etaxonomy.cdm.api.service.config.FindTaxaAndNamesConfiguratorImpl;
  37 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  38 import eu.etaxonomy.cdm.api.service.pager.Pager;
  39 import eu.etaxonomy.cdm.api.service.search.ICdmMassIndexer;
  40 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  41 import eu.etaxonomy.cdm.api.service.search.LuceneParseException;
  42 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  43 import eu.etaxonomy.cdm.common.UTF8;
  44 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  45 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  46 import eu.etaxonomy.cdm.model.common.CdmBase;
  47 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  48 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  49 import eu.etaxonomy.cdm.model.common.Language;
  50 import eu.etaxonomy.cdm.model.description.CategoricalData;
  51 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  52 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  53 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  54 import eu.etaxonomy.cdm.model.description.Distribution;
  55 import eu.etaxonomy.cdm.model.description.Feature;
  56 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  57 import eu.etaxonomy.cdm.model.description.State;
  58 import eu.etaxonomy.cdm.model.description.StateData;
  59 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  60 import eu.etaxonomy.cdm.model.description.TextData;
  61 import eu.etaxonomy.cdm.model.location.Country;
  62 import eu.etaxonomy.cdm.model.location.NamedArea;
  63 import eu.etaxonomy.cdm.model.name.IBotanicalName;
  64 import eu.etaxonomy.cdm.model.name.Rank;
  65 import eu.etaxonomy.cdm.model.name.TaxonName;
  66 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  67 import eu.etaxonomy.cdm.model.reference.Reference;
  68 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  69 import eu.etaxonomy.cdm.model.taxon.Classification;
  70 import eu.etaxonomy.cdm.model.taxon.Synonym;
  71 import eu.etaxonomy.cdm.model.taxon.SynonymType;
  72 import eu.etaxonomy.cdm.model.taxon.Taxon;
  73 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  74 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  75 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  76 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  77 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  78 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  79 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  80 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  81
  82 /**
  83  * @author a.babadshanjan, a.kohlbecker
  84  * @since 04.02.2009
  85  */
  86 public class TaxonServiceSearchTest extends CdmTransactionalIntegrationTest {
  87
  88     private static final String ABIES_BALSAMEA_UUID = "f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8";
  89
  90     private static final String ABIES_ALBA_UUID = "7dbd5810-a3e5-44b6-b563-25152b8867f4";
  91
  92     private static final String CLASSIFICATION_UUID = "2a5ceebb-4830-4524-b330-78461bf8cb6b";
  93
  94     private static final String CLASSIFICATION_ALT_UUID = "d7c741e3-ae9e-4a7d-a566-9e3a7a0b51ce";
  95
  96     private static final String D_ABIES_BALSAMEA_UUID = "900108d8-e6ce-495e-b32e-7aad3099135e";
  97
  98     private static final String D_ABIES_ALBA_UUID = "ec8bba03-d993-4c85-8472-18b14942464b";
  99
 100     private static final String D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID = "e9d8c2fd-6409-46d5-9c2e-14a2bbb1b2b1";
 101
 102     private static final int NUM_OF_NEW_RADOM_ENTITIES = 1000;
 103
 104     private static Logger logger = Logger.getLogger(TaxonServiceSearchTest.class);
 105
 106
 107
 108     @SpringBeanByType
 109     private ITaxonService taxonService;
 110     @SpringBeanByType
 111     private ITermService termService;
 112     @SpringBeanByType
 113     private IClassificationService classificationService;
 114     @SpringBeanByType
 115     private IReferenceService referenceService;
 116     @SpringBeanByType
 117     private IDescriptionService descriptionService;
 118     @SpringBeanByType
 119     private INameService nameService;
 120     @SpringBeanByType
 121     private ICdmMassIndexer indexer;
 122
 123     @SpringBeanByType
 124     private ITaxonNodeService nodeService;
 125
 126     private static final int BENCHMARK_ROUNDS = 300;
 127
 128     private Set<Class<? extends CdmBase>> typesToIndex = null;
 129
 130     private NamedArea germany;
 131     private NamedArea france ;
 132     private NamedArea russia ;
 133     private NamedArea canada ;
 134
 135     @Before
 136     public void setUp() throws Exception {
 137         typesToIndex = new HashSet<Class<? extends CdmBase>>();
 138         typesToIndex.add(DescriptionElementBase.class);
 139         typesToIndex.add(TaxonBase.class);
 140         typesToIndex.add(TaxonRelationship.class);
 141
 142         germany =  Country.GERMANY();
 143         france = Country.FRANCEFRENCHREPUBLIC();
 144         russia = Country.RUSSIANFEDERATION();
 145         canada = Country.CANADA();
 146
 147     }
 148
 149     @Test
 150     public void testDbUnitUsageTest() throws Exception {
 151         assertNotNull("taxonService should exist", taxonService);
 152         assertNotNull("nameService should exist", nameService);
 153     }
 154
 155
 156     @SuppressWarnings("rawtypes")
 157     @Test
 158     @DataSet
 159     public final void testPurgeAndReindex() throws IOException, LuceneParseException {
 160
 161         refreshLuceneIndex();
 162
 163         Pager<SearchResult<TaxonBase>> pager;
 164
 165         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 8
 166         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 167
 168         indexer.purge(null);
 169         commitAndStartNewTransaction(null);
 170
 171         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 0
 172         Assert.assertEquals("Expecting no entities since the index has been purged", 0, pager.getCount().intValue());
 173
 174         indexer.reindex(indexer.indexedClasses(), null);
 175         commitAndStartNewTransaction(null);
 176
 177         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 8
 178         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 179     }
 180
 181
 182     @SuppressWarnings("rawtypes")
 183     @Test
 184     @DataSet
 185     public final void testFindByDescriptionElementFullText_CommonName() throws IOException,
 186             LuceneParseException {
 187
 188         refreshLuceneIndex();
 189
 190         Pager<SearchResult<TaxonBase>> pager;
 191
 192         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null, null,
 193                 false, null, null, null, null);
 194         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1,
 195                 pager.getCount().intValue());
 196
 197         // the description containing the Nulltanne has no taxon attached,
 198         // taxon.id = null
 199         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Nulltanne", null, null, null,
 200                 false, null, null, null, null);
 201         Assert.assertEquals("Expecting no entity when searching for 'Nulltanne' ", 0, pager.getCount().intValue());
 202
 203         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 204                 Arrays.asList(new Language[] { Language.GERMAN() }), false, null, null, null, null);
 205         Assert.assertEquals("Expecting one entity when searching in German", 1, pager.getCount().intValue());
 206
 207         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 208                 Arrays.asList(new Language[] { Language.RUSSIAN() }), false, null, null, null, null);
 209         Assert.assertEquals("Expecting no entity when searching in Russian", 0, pager.getCount().intValue());
 210
 211     }
 212
 213     @SuppressWarnings("rawtypes")
 214     @Test
 215     @DataSet
 216     public final void testFindByDescriptionElementFullText_Distribution() throws IOException, LuceneParseException {
 217
 218         refreshLuceneIndex();
 219
 220         Pager<SearchResult<TaxonBase>> pager;
 221         // by Area
 222         pager = taxonService.findByDescriptionElementFullText(null, "Canada", null, null, null, false, null, null, null, null);
 223         Assert.assertEquals("Expecting one entity when searching for arae 'Canada'", 1, pager.getCount().intValue());
 224         // by Status
 225         pager = taxonService.findByDescriptionElementFullText(null, "present", null, null, null, false, null, null, null, null);
 226         Assert.assertEquals("Expecting one entity when searching for status 'present'", 1, pager.getCount().intValue());
 227     }
 228
 229     @SuppressWarnings("rawtypes")
 230     @Test
 231     @DataSet
 232     public final void testFindByDescriptionElementFullText_wildcard() throws IOException, LuceneParseException {
 233
 234         refreshLuceneIndex();
 235
 236         Pager<SearchResult<TaxonBase>> pager;
 237
 238         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
 239         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1, pager.getCount().intValue());
 240     }
 241
 242     /**
 243      * Regression test for #3113 (hibernate search: wildcard query can cause BooleanQuery$TooManyClauses: maxClauseCount is set to 1024)
 244      *
 245      * @throws IOException
 246      * @throws LuceneParseException
 247      */
 248     @SuppressWarnings("rawtypes")
 249     @Test
 250     @DataSet
 251     public final void testFindByDescriptionElementFullText_TooManyClauses() throws IOException, LuceneParseException {
 252
 253         // generate 1024 terms to reproduce the bug
 254         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 255         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 256         while(uniqueRandomStrs.size() < 1024){
 257             uniqueRandomStrs.add(RandomStringUtils.random(10, true, false));
 258         }
 259         for(String rndStr: uniqueRandomStrs){
 260             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 261         }
 262         descriptionService.saveOrUpdate(description);
 263         commitAndStartNewTransaction(null);
 264
 265         refreshLuceneIndex();
 266
 267         Pager<SearchResult<TaxonBase>> pager;
 268
 269         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, null, null, null, null);
 270         Assert.assertEquals("Expecting all 1024 entities grouped into one SearchResult item when searching for Rot*", 1, pager.getCount().intValue());
 271     }
 272
 273     /**
 274      * Regression test for #3116 (fulltext search: always only one page of results)
 275      *
 276      * @throws IOException
 277      * @throws LuceneParseException
 278      */
 279     @SuppressWarnings("rawtypes")
 280     @Test
 281     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 282     @Ignore
 283     public final void testFullText_Paging() throws IOException, LuceneParseException {
 284
 285         Reference sec = ReferenceFactory.newDatabase();
 286         referenceService.save(sec);
 287
 288         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 289         int numOfItems = 100;
 290         while(uniqueRandomStrs.size() < numOfItems){
 291             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 292         }
 293
 294         for(String rndStr: uniqueRandomStrs){
 295
 296             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(Rank.SERIES()), sec);
 297             taxon.setTitleCache("Tax" + rndStr, true);
 298             taxonService.save(taxon);
 299
 300             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 301             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 302             descriptionService.saveOrUpdate(description);
 303         }
 304
 305         commitAndStartNewTransaction(new String[]{"TAXONBASE"});
 306         refreshLuceneIndex();
 307
 308         int pageSize = 10;
 309
 310         Pager<SearchResult<TaxonBase>> pager;
 311
 312         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, null, null, null);
 313         Assert.assertEquals("unexpeted number of pages", Integer.valueOf(numOfItems / pageSize), pager.getPagesAvailable());
 314         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 9, null, null);
 315         Assert.assertNotNull("last page must have records", pager.getRecords());
 316         Assert.assertNotNull("last item on last page must exist", pager.getRecords().get(0));
 317         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 10, null, null);
 318         Assert.assertNotNull("last page + 1 must not have any records", pager.getRecords());
 319     }
 320
 321     /**
 322      * test for max score and sort by score of hit groups
 323      * with all matches per taxon in a single TextData  element
 324      * see {@link #testFullText_ScoreAndOrder_2()} for the complement
 325      * test with matches in multiple TextData per taxon
 326      *
 327      * @throws IOException
 328      * @throws LuceneParseException
 329      */
 330     @SuppressWarnings("rawtypes")
 331     @Test
 332     @DataSet
 333     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 334     public final void testFullText_ScoreAndOrder_1() throws IOException, LuceneParseException {
 335
 336         int numOfTaxa = 3;
 337
 338         UUID[] taxonUuids = new UUID[numOfTaxa];
 339         StringBuilder text = new StringBuilder();
 340
 341         for(int i = 0; i < numOfTaxa; i++){
 342
 343             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 344             taxon.setTitleCache("Taxon_" + i, true);
 345             taxonUuids[i] = taxon.getUuid();
 346             taxonService.save(taxon);
 347
 348             text.append(" ").append("Rot");
 349             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 350             description.addElement(TextData.NewInstance(text.toString(), Language.DEFAULT(), null));
 351             descriptionService.saveOrUpdate(description);
 352         }
 353
 354         commitAndStartNewTransaction(null);
 355         refreshLuceneIndex();
 356
 357         Pager<SearchResult<TaxonBase>> pager;
 358
 359         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 360         for(int i = 0; i < numOfTaxa; i++){
 361             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 362         }
 363         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 364     }
 365
 366     /**
 367      * test for max score and sort by score of hit groups
 368      * with all matches per taxon in a multiple TextData elements
 369      * see {@link #testFullText_ScoreAndOrder_1()} for the complement
 370      * test with matches in a single TextData per taxon
 371      *
 372      * @throws IOException
 373      * @throws LuceneParseException
 374      */
 375     @SuppressWarnings("rawtypes")
 376     @Test
 377     @DataSet
 378     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 379     public final void testFullText_ScoreAndOrder_2() throws IOException, LuceneParseException {
 380
 381         int numOfTaxa = 3;
 382
 383         UUID[] taxonUuids = new UUID[numOfTaxa];
 384
 385         for(int i = 0; i < numOfTaxa; i++){
 386
 387             Taxon taxon = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(null), null);
 388             taxon.setTitleCache("Taxon_" + i, true);
 389             taxonUuids[i] = taxon.getUuid();
 390             taxonService.save(taxon);
 391
 392             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 393             for(int k = 0; k < i; k++){
 394                 description.addElement(TextData.NewInstance("Rot", Language.DEFAULT(), null));
 395             }
 396             descriptionService.saveOrUpdate(description);
 397         }
 398
 399         commitAndStartNewTransaction(null);
 400         refreshLuceneIndex();
 401
 402         Pager<SearchResult<TaxonBase>> pager;
 403
 404         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 405         for(int i = 0; i < numOfTaxa; i++){
 406             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 407         }
 408         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 409     }
 410
 411
 412     /**
 413      * @throws IOException
 414      * @throws LuceneParseException
 415      * @throws LuceneMultiSearchException
 416      */
 417     @Test
 418     @DataSet
 419     public final void testFullText_Grouping() throws IOException, LuceneParseException, LuceneMultiSearchException {
 420
 421         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 422         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 423         int numOfItems = 100;
 424         while(uniqueRandomStrs.size() < numOfItems){
 425             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 426         }
 427         for(String rndStr: uniqueRandomStrs){
 428             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 429         }
 430         descriptionService.saveOrUpdate(description);
 431
 432         commitAndStartNewTransaction(new String[]{"DESCRIPTIONELEMENTBASE"});
 433
 434         refreshLuceneIndex();
 435
 436         int pageSize = 10;
 437
 438         Pager<SearchResult<TaxonBase>> pager;
 439         boolean highlightFragments = true;
 440
 441         // test with findByDescriptionElementFullText
 442         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, highlightFragments, pageSize, null, null, null);
 443         logFreeTextSearchResults(pager, Level.DEBUG, null);
 444         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 445         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 446         Map<String, String[]> highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 447         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 448         int maxDocsPerGroup = 10;
 449         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 450
 451         // test with findByEverythingFullText
 452         pager = taxonService.findByEverythingFullText( "Rot*", null, null, highlightFragments, pageSize, null, null, null);
 453         logFreeTextSearchResults(pager, Level.DEBUG, null);
 454         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 455         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 456         highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 457         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 458         maxDocsPerGroup = 10;
 459         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 460
 461     }
 462
 463     @SuppressWarnings("rawtypes")
 464     @Test
 465     @DataSet
 466     @Ignore
 467     public final void testFindByDescriptionElementFullText_TextData() throws IOException, LuceneParseException {
 468
 469         refreshLuceneIndex();
 470
 471         Pager<SearchResult<TaxonBase>> pager;
 472         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, false, null, null, null, null);
 473         logFreeTextSearchResults(pager, Level.DEBUG, null);
 474         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 475         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 476         Assert.assertTrue("Expecting two docs, one for RUSSIAN and one for GERMAN", pager.getRecords().get(0).getDocs().size() == 2);
 477         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 478
 479
 480         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, null, null, false, null, null, null, null);
 481         Assert.assertEquals("Expecting one entity when searching for any type", 1, pager.getCount().intValue());
 482
 483         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.UNKNOWN()}), null, false, null, null, null, null);
 484         Assert.assertEquals("Expecting one entity when searching for any type and for Feature DESCRIPTION", 1, pager.getCount().intValue());
 485
 486         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER()}), null, false, null, null, null, null);
 487         Assert.assertEquals("Expecting no entity when searching for any type and for Feature CHROMOSOME_NUMBER", 0, pager.getCount().intValue());
 488
 489         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER(), Feature.UNKNOWN()}), null, false, null, null, null, null);
 490         Assert.assertEquals("Expecting no entity when searching for any type and for Feature DESCRIPTION or CHROMOSOME_NUMBER", 1, pager.getCount().intValue());
 491
 492         pager = taxonService.findByDescriptionElementFullText(Distribution.class, "Abies", null, null, null, false, null, null, null, null);
 493         Assert.assertEquals("Expecting no entity when searching for Distribution", 0, pager.getCount().intValue());
 494
 495         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{}), false, null, null, null, null);
 496         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 497         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 498
 499         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.RUSSIAN()}), false, null, null, null, null);
 500         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 501         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 502
 503         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 504         Assert.assertEquals("Expecting no entity", 0, pager.getCount().intValue());
 505
 506         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 507         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 508         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 509     }
 510
 511     @SuppressWarnings("rawtypes")
 512     @Test
 513     @DataSet
 514     public final void testFindByDescriptionElementFullText_MultipleWords() throws IOException, LuceneParseException {
 515
 516         refreshLuceneIndex();
 517
 518         // Pflanzenart aus der Gattung der Tannen
 519         long start = System.currentTimeMillis();
 520
 521         Pager<SearchResult<TaxonBase>> pager;
 522         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, false, null, null, null, null);
 523         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 524
 525         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Wespen", null, null, null, false, null, null, null, null);
 526         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 527
 528         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, false, null, null, null, null);
 529         Assert.assertEquals("AND search : Expecting one entity", 1, pager.getCount().intValue());
 530
 531         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Wespen", null, null, null, false, null, null, null, null);
 532         Assert.assertEquals("AND search : Expecting no entity", 0, pager.getCount().intValue());
 533
 534         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, false, null, null, null, null);
 535         Assert.assertEquals("Phrase search : Expecting one entity", 1, pager.getCount().intValue());
 536
 537         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Wespen\"", null, null, null, false, null, null, null, null);
 538         Assert.assertEquals("Phrase search : Expecting one entity", 0, pager.getCount().intValue());
 539
 540         logger.info("testFindByDescriptionElementFullText_MultipleWords() duration: " + (System.currentTimeMillis() - start) + "ms");
 541
 542     }
 543
 544
 545     @SuppressWarnings("rawtypes")
 546     @Test
 547     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 548     public final void testFindByDescriptionElementFullText_modify_DescriptionElement() throws IOException, LuceneParseException {
 549
 550         refreshLuceneIndex();
 551
 552         Pager<SearchResult<TaxonBase>> pager;
 553         //
 554         // modify the DescriptionElement
 555         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 556         Assert.assertTrue("Search did not return any results", pager.getRecords().size() > 0);
 557         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 558         Document indexDocument = pager.getRecords().get(0).getDocs().iterator().next();
 559         String[] descriptionElementUuidStr = indexDocument.getValues("uuid");
 560         String[] inDescriptionUuidStr = indexDocument.getValues("inDescription.uuid");
 561         // is only one uuid!
 562         DescriptionElementBase textData = descriptionService.getDescriptionElementByUuid(UUID.fromString(descriptionElementUuidStr[0]));
 563
 564         ((TextData)textData).removeText(Language.GERMAN());
 565         ((TextData)textData).putText(Language.SPANISH_CASTILIAN(), "abeto bals"+UTF8.SMALL_A_ACUTE+"mico");
 566
 567         descriptionService.saveDescriptionElement(textData);
 568         commitAndStartNewTransaction(null);
 569 //        printDataSet(System.out, new String[] {
 570 //                "DESCRIPTIONELEMENTBASE", "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING" }
 571 //        );
 572
 573         //
 574         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 575         Assert.assertEquals("The german 'Balsam-Tanne' TextData should no longer be indexed", 0, pager.getCount().intValue());
 576         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 577         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 578         pager = taxonService.findByDescriptionElementFullText(TextData.class, "bals"+UTF8.SMALL_A_ACUTE+"mico", null, null, null, false, null, null, null, null);
 579         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 580
 581         //
 582         // modify the DescriptionElement via the Description object
 583         DescriptionBase<?> description = descriptionService.find(UUID.fromString(inDescriptionUuidStr[0]));
 584         Set<DescriptionElementBase> elements = description.getElements();
 585         for( DescriptionElementBase elm : elements){
 586             if(elm.getUuid().toString().equals(descriptionElementUuidStr[0])){
 587                 ((TextData)elm).removeText(Language.SPANISH_CASTILIAN());
 588                 ((TextData)elm).putText(Language.POLISH(), "Jod"+UTF8.POLISH_L+"a balsamiczna");
 589             }
 590         }
 591         descriptionService.saveOrUpdate(description);
 592         commitAndStartNewTransaction(null);
 593         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 594         Assert.assertEquals("The spanish 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico' TextData should no longer be indexed", 0, pager.getCount().intValue());
 595         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsamiczna", null, null, Arrays.asList(new Language[]{Language.POLISH()}), false, null, null, null, null);
 596         Assert.assertEquals("expecting to find the POLISH 'Jod"+UTF8.POLISH_L+"a balsamiczna'", 1, pager.getCount().intValue());
 597     }
 598
 599     @SuppressWarnings("rawtypes")
 600     @Test
 601     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 602     public final void testFindByDescriptionElementFullText_modify_Taxon() throws IOException, LuceneParseException {
 603
 604         refreshLuceneIndex();
 605
 606         Pager<SearchResult<TaxonBase>> pager;
 607         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
 608         TaxonDescription d_abies_balsamea = (TaxonDescription)descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 609
 610         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 611         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne'", 1, pager.getCount().intValue());
 612
 613         // exchange the Taxon with another one via the Taxon object
 614         // 1.) remove existing description:
 615         t_abies_balsamea.removeDescription(d_abies_balsamea);
 616
 617         taxonService.saveOrUpdate(t_abies_balsamea);
 618         commitAndStartNewTransaction(null);
 619
 620         t_abies_balsamea = (Taxon)taxonService.find(t_abies_balsamea.getUuid());
 621
 622         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 623         Assert.assertEquals("'Balsam-Tanne' should no longer be found", 0, pager.getCount().intValue());
 624
 625         // 2.) create new description and add to taxon:
 626         TaxonDescription d_abies_balsamea_new = TaxonDescription.NewInstance();
 627         d_abies_balsamea_new
 628                 .addElement(TextData
 629                         .NewInstance(
 630                                 "Die Balsamtanne ist mit bis zu 30 m Höhe ein mittelgro"+UTF8.SHARP_S+"er Baum und kann bis zu 200 Jahre alt werden",
 631                                 Language.GERMAN(), null));
 632         t_abies_balsamea.addDescription(d_abies_balsamea_new);
 633         // set authorshipCache to null to avoid validation exception,
 634         // this is maybe not needed in future,  see ticket #3344
 635         IBotanicalName abies_balsamea = CdmBase.deproxy(t_abies_balsamea.getName(), TaxonName.class);
 636         abies_balsamea.setAuthorshipCache(null);
 637         printDataSet(System.err, new String[] {"LANGUAGESTRING_AUD"});
 638         taxonService.saveOrUpdate(t_abies_balsamea);
 639         commitAndStartNewTransaction(null);
 640
 641 //        printDataSet(System.out, new String[] {
 642 //                "DESCRIPTIONBASE"
 643 //        });
 644
 645         pager = taxonService.findByDescriptionElementFullText(TextData.class, "mittelgro"+UTF8.SHARP_S+"er Baum", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 646         Assert.assertEquals("the taxon should be found via the new Description", 1, pager.getCount().intValue());
 647     }
 648
 649     @SuppressWarnings("rawtypes")
 650     @Test
 651     @DataSet
 652     public final void testFindByDescriptionElementFullText_modify_Classification() throws IOException, LuceneParseException {
 653
 654         refreshLuceneIndex();
 655
 656         Pager<SearchResult<TaxonBase>> pager;
 657
 658         // put taxon into other classification, new taxon node
 659         Classification classification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 660         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 661
 662         // TODO: why is the test failing when the childNode is already retrieved here, and not after the following four lines?
 663         //TaxonNode childNode = classification.getChildNodes().iterator().next();
 664
 665         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 666         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne' even if filtering by classification", 1, pager.getCount().intValue());
 667         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 668         Assert.assertEquals("GERMAN 'Balsam-Tanne' should NOT be found in other classification", 0, pager.getCount().intValue());
 669
 670         // check for the right taxon node
 671         TaxonNode childNode = classification.getChildNodes().iterator().next();
 672         Assert.assertEquals("expecting Abies balsamea sec.", childNode.getTaxon().getUuid().toString(), ABIES_BALSAMEA_UUID);
 673         Assert.assertEquals("expecting default classification", childNode.getClassification().getUuid().toString(), CLASSIFICATION_UUID);
 674
 675         // moving the taxon around, the rootnode is only a proxy
 676         alternateClassification.setRootNode(HibernateProxyHelper.deproxy(alternateClassification.getRootNode(), TaxonNode.class));
 677         alternateClassification.addChildNode(childNode, null, null);
 678
 679         classificationService.saveOrUpdate(alternateClassification);
 680         commitAndStartNewTransaction(null);
 681
 682 //        printDataSet(System.out, new String[] {
 683 //            "TAXONBASE", "TAXONNODE", "CLASSIFICATION"
 684 //        });
 685
 686         // reload classification
 687         classification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 688         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 689         Assert.assertEquals("GERMAN 'Balsam-Tanne' should now be found in other classification", 1, pager.getCount().intValue());
 690
 691         classification.getChildNodes().clear();
 692         classificationService.saveOrUpdate(classification);
 693         commitAndStartNewTransaction(null);
 694
 695         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", classification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 696         Assert.assertEquals("Now the GERMAN 'Balsam-Tanne' should NOT be found in original classification", 0, pager.getCount().intValue());
 697
 698     }
 699
 700     @SuppressWarnings("rawtypes")
 701     @Test
 702     @DataSet
 703     public final void testFindByDescriptionElementFullText_CategoricalData() throws IOException, LuceneParseException {
 704
 705         // add CategoricalData
 706         DescriptionBase d_abies_balsamea = descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 707         // Categorical data
 708         CategoricalData cdata = CategoricalData.NewInstance();
 709         cdata.setFeature(Feature.DESCRIPTION());
 710         State state = State.NewInstance("green", "green", "gn");
 711
 712         StateData statedata = StateData.NewInstance(state);
 713         statedata.putModifyingText(Language.ENGLISH(), "always, even during winter");
 714         cdata.addStateData(statedata);
 715         d_abies_balsamea.addElement(cdata);
 716
 717         UUID termUUID = termService.save(state).getUuid();
 718         descriptionService.save(d_abies_balsamea);
 719
 720         commitAndStartNewTransaction(null);
 721
 722 //        printDataSet(System.out, new String[] {
 723 //                 "STATEDATA", "STATEDATA_DEFINEDTERMBASE", "STATEDATA_LANGUAGESTRING", "LANGUAGESTRING"});
 724
 725         refreshLuceneIndex();
 726
 727         Pager<SearchResult<TaxonBase>> pager;
 728         pager = taxonService.findByDescriptionElementFullText(CategoricalData.class, "green", null, null, null, false, null, null, null, null);
 729         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 730         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 731         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 732         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 733
 734
 735         //TODO modify the StateData
 736         TaxonBase taxon = pager.getRecords().get(0).getEntity();
 737
 738         String newName = "Quercus robur";
 739         taxon.setTitleCache(newName + " sec. ", true);
 740
 741         taxonService.saveOrUpdate(taxon);
 742         commitAndStartNewTransaction(null);
 743
 744         taxon = taxonService.find(taxon.getUuid());
 745         Assert.assertEquals(newName + " sec. ", taxon.getTitleCache());
 746         DefinedTermBase term = termService.find(termUUID);
 747
 748         termService.delete(term);
 749
 750     }
 751
 752     @SuppressWarnings("rawtypes")
 753     @Test
 754     @DataSet
 755     public final void testFindByDescriptionElementFullText_Highlighting() throws IOException, LuceneParseException {
 756
 757         refreshLuceneIndex();
 758
 759         Pager<SearchResult<TaxonBase>> pager;
 760         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, true, null, null, null, null);
 761         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 762         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 763         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
 764         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 765         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Abies</B>"));
 766
 767         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, true, null, null, null, null);
 768         searchResult = pager.getRecords().get(0);
 769         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 770         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 771         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") || fragments[0].contains("<B>Tannen</B>"));
 772
 773         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, true, null, null, null, null);
 774         searchResult = pager.getRecords().get(0);
 775         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 776         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 777         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") && fragments[0].contains("<B>Tannen</B>"));
 778
 779         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, true, null, null, null, null);
 780         searchResult = pager.getRecords().get(0);
 781         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 782         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 783         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B> <B>aus</B> <B>der</B> <B>Gattung</B> <B>der</B> <B>Tannen</B>"));
 784
 785         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Gatt*", null, null, null, true, null, null, null, null);
 786         searchResult = pager.getRecords().get(0);
 787         Assert.assertTrue("Wildcard search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 788         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 789         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Gatt"));
 790     }
 791
 792
 793     @Test
 794     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 795     public final void testFindByFullText() throws IOException, LuceneParseException {
 796
 797         refreshLuceneIndex();
 798
 799         Classification europeanAbiesClassification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 800
 801         Pager<SearchResult<TaxonBase>> pager;
 802
 803         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 7
 804         logFreeTextSearchResults(pager, Level.DEBUG, null);
 805         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 806
 807         pager = taxonService.findByFullText(Taxon.class, "Abies", null, null, true, null, null, null, null); // --> 6
 808         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 809
 810         pager = taxonService.findByFullText(Synonym.class, "Abies", null, null, true, null, null, null, null); // --> 1
 811         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 812
 813         pager = taxonService.findByFullText(TaxonBase.class, "sec", null, null, true, null, null, null, null); // --> 7
 814         Assert.assertEquals("Expecting 8 entities", 9, pager.getCount().intValue());
 815
 816         pager = taxonService.findByFullText(null, "genus", null, null, true, null, null, null, null); // --> 1
 817         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 818
 819         pager = taxonService.findByFullText(Taxon.class, "subalpina", null, null, true, null, null, null, null); // --> 0
 820         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 821
 822         // synonym in classification ???
 823     }
 824
 825     @Test
 826     @DataSet
 827     public final void testPrepareByAreaSearch() throws IOException, LuceneParseException {
 828
 829         List<PresenceAbsenceTerm> statusFilter = new ArrayList<PresenceAbsenceTerm>();
 830         List<NamedArea> areaFilter = new ArrayList<NamedArea>();
 831         areaFilter.add(germany);
 832         areaFilter.add(canada);
 833         areaFilter.add(russia);
 834
 835         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDistribution(areaFilter, statusFilter, null, 20, 0, null, null);
 836         Assert.assertEquals("Expecting 2 entities", Integer.valueOf(2), Integer.valueOf(pager.getRecords().size()));
 837
 838     }
 839
 840     @Test
 841     @DataSet
 842     public final void testFindTaxaAndNamesByFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
 843
 844         refreshLuceneIndex();
 845
 846         Pager<SearchResult<TaxonBase>> pager;
 847
 848         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 849
 850
 851         pager = taxonService.findTaxaAndNamesByFullText(
 852                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 853                 "Abies", null, null, null, null, true, null, null, null, null);
 854 //        logPagerRecords(pager, Level.DEBUG);
 855         Assert.assertEquals("doTaxa & doSynonyms", 8, pager.getCount().intValue());
 856
 857         pager = taxonService.findTaxaAndNamesByFullText(
 858                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 859                 "Abies", null, null, null, null, true, null, null, null, null);
 860 //        logPagerRecords(pager, Level.DEBUG);
 861         Assert.assertEquals("all search modes", 8, pager.getCount().intValue());
 862
 863         pager = taxonService.findTaxaAndNamesByFullText(
 864                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 865                 "Abies", alternateClassification, null, null, null, true, null, null, null, null);
 866 //        logPagerRecords(pager, Level.DEBUG);
 867         Assert.assertEquals("all search modes, filtered by alternateClassification", 1, pager.getCount().intValue());
 868
 869         pager = taxonService.findTaxaAndNamesByFullText(
 870                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
 871                 "Abies", null, null, null, null, true, null, null, null, null);
 872         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 873         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 874         Assert.assertEquals(Synonym.class, searchResult.getEntity().getClass());
 875         // Abies subalpina sec. Kohlbecker, A., Testcase standart views, 2013
 876
 877
 878         pager = taxonService.findTaxaAndNamesByFullText(
 879                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 880                 "Abies", null, null, null, null, true, null, null, null, null);
 881         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 882
 883
 884         pager = taxonService.findTaxaAndNamesByFullText(
 885                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 886                 "Tanne", null, null, null, null, true, null, null, null, null);
 887         Assert.assertEquals("Expecting 1 entity", 1, pager.getRecords().size());
 888         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 889
 890         pager = taxonService.findTaxaAndNamesByFullText(
 891                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
 892                 "kawakamii", null, null, null, null, true, null, null, null, null);
 893         logFreeTextSearchResults(pager, Level.DEBUG, null);
 894         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 895
 896     }
 897
 898     @Test
 899     @DataSet
 900     public final void testFindTaxaAndNamesByFullText_PhraseQuery() throws IOException, LuceneParseException, LuceneMultiSearchException {
 901
 902         refreshLuceneIndex();
 903
 904         Pager<SearchResult<TaxonBase>> pager;
 905
 906
 907         pager = taxonService.findTaxaAndNamesByFullText(
 908                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 909                 "\"Abies alba\"", null, null, null, null, true, null, null, null, null);
 910 //        logPagerRecords(pager, Level.DEBUG);
 911         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 1, pager.getCount().intValue());
 912
 913         pager = taxonService.findTaxaAndNamesByFullText(
 914                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 915                 "\"Abies al*\"", null, null, null, null, true, null, null, null, null);
 916 //        logPagerRecords(pager, Level.DEBUG);
 917         Assert.assertEquals("doTaxa & doSynonyms with complex phrase query", 1, pager.getCount().intValue());
 918
 919         pager = taxonService.findTaxaAndNamesByFullText(
 920                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 921                 "\"Abies*\"", null, null, null, null, true, null, null, null, null);
 922 //        logPagerRecords(pager, Level.DEBUG);
 923         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 8, pager.getCount().intValue());
 924
 925     }
 926
 927     @Test
 928     @DataSet
 929     public final void testFindTaxaAndNamesByFullText_Sort() throws IOException, LuceneParseException, LuceneMultiSearchException {
 930
 931         refreshLuceneIndex();
 932
 933         Pager<SearchResult<TaxonBase>> pager;
 934
 935         List<OrderHint> orderHints = new ArrayList<OrderHint>();
 936
 937         String[] docFields2log = new String[]{"id"};
 938
 939         // SortById
 940         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
 941         pager = taxonService.findTaxaAndNamesByFullText(
 942                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
 943                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 944 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 945         int lastId = -1;
 946         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 947             if(lastId != -1){
 948                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
 949             }
 950             lastId = rs.getEntity().getId();
 951         }
 952
 953         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
 954         pager = taxonService.findTaxaAndNamesByFullText(
 955                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 956                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 957 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 958
 959         lastId = -1;
 960         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 961             if(lastId != -1){
 962                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
 963             }
 964             lastId = rs.getEntity().getId();
 965         }
 966
 967         // Sortby NOMENCLATURAL_SORT_ORDER TODO make assertions !!!
 968         orderHints.clear();
 969         orderHints.addAll(OrderHint.NOMENCLATURAL_SORT_ORDER.asList());
 970         pager = taxonService.findTaxaAndNamesByFullText(
 971                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 972                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 973         logFreeTextSearchResults(pager, Level.DEBUG, null);
 974
 975     }
 976
 977     @Test
 978     @DataSet
 979     public final void testFindTaxaAndNamesByFullText_AreaFilter() throws IOException, LuceneParseException, LuceneMultiSearchException {
 980
 981         refreshLuceneIndex();
 982
 983         Pager<SearchResult<TaxonBase>> pager;
 984
 985         Set<NamedArea> a_germany_canada_russia = new HashSet<NamedArea>();
 986         a_germany_canada_russia.add(germany);
 987         a_germany_canada_russia.add(canada);
 988         a_germany_canada_russia.add(russia);
 989
 990         Set<NamedArea> a_russia = new HashSet<NamedArea>();
 991         a_russia.add(russia);
 992
 993         Set<PresenceAbsenceTerm> present = new HashSet<PresenceAbsenceTerm>();
 994         present.add(PresenceAbsenceTerm.PRESENT());
 995
 996         Set<PresenceAbsenceTerm> present_native = new HashSet<PresenceAbsenceTerm>();
 997         present_native.add(PresenceAbsenceTerm.PRESENT());
 998         present_native.add(PresenceAbsenceTerm.NATIVE());
 999
1000         Set<PresenceAbsenceTerm> absent = new HashSet<PresenceAbsenceTerm>();
1001         absent.add(PresenceAbsenceTerm.ABSENT());
1002
1003         pager = taxonService.findTaxaAndNamesByFullText(
1004                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
1005                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1006         logFreeTextSearchResults(pager, Level.DEBUG, null);
1007
1008         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1009         pager = taxonService.findTaxaAndNamesByFullText(
1010                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
1011                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1012         Assert.assertEquals("synonyms with matching area filter", 1, pager.getCount().intValue());
1013
1014         pager = taxonService.findTaxaAndNamesByFullText(
1015                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1016                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1017         logFreeTextSearchResults(pager, Level.DEBUG, null);
1018         Assert.assertEquals("taxa and synonyms with matching area filter", 3, pager.getCount().intValue());
1019
1020         pager = taxonService.findTaxaAndNamesByFullText(
1021                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1022                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1023         Assert.assertEquals("taxa and synonyms with matching area & status filter 1", 3, pager.getCount().intValue());
1024
1025         pager = taxonService.findTaxaAndNamesByFullText(
1026                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1027                 "Abies", null, a_germany_canada_russia, present, null, true, null, null, null, null);
1028         Assert.assertEquals("taxa and synonyms with matching area & status filter 2", 2, pager.getCount().intValue());
1029
1030         pager = taxonService.findTaxaAndNamesByFullText(
1031                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1032                 "Abies", null, a_russia, present, null, true, null, null, null, null);
1033         Assert.assertEquals("taxa and synonyms with non matching area & status filter", 0, pager.getCount().intValue());
1034
1035         pager = taxonService.findTaxaAndNamesByFullText(
1036                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
1037                 "Tanne", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1038         Assert.assertEquals("ByCommonNames with area filter", 1, pager.getCount().intValue());
1039
1040         // abies_kawakamii_sensu_komarov as misapplied name for t_abies_balsamea
1041         pager = taxonService.findTaxaAndNamesByFullText(
1042                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1043                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1044         Assert.assertEquals("misappliedNames with matching area & status filter", 1, pager.getCount().intValue());
1045
1046
1047         // 1. remove existing taxon relation
1048         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
1049         Set<TaxonRelationship> relsTo = t_abies_balsamea.getRelationsToThisTaxon();
1050         Assert.assertEquals(1, relsTo.size());
1051         TaxonRelationship taxonRelation = relsTo.iterator().next();
1052         t_abies_balsamea.removeTaxonRelation(taxonRelation);
1053         taxonService.saveOrUpdate(t_abies_balsamea);
1054         commitAndStartNewTransaction(null);
1055
1056         pager = taxonService.findTaxaAndNamesByFullText(
1057                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1058                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1059         Assert.assertEquals("misappliedNames with matching area & status filter, should match nothing now", 0, pager.getCount().intValue());
1060
1061         // 2. now add abies_kawakamii_sensu_komarov as misapplied name for t_abies_alba and search for misapplications in russia: ABSENT
1062         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(UUID.fromString(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1063         Taxon t_abies_alba = (Taxon)taxonService.find(UUID.fromString(ABIES_ALBA_UUID));
1064         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1065
1066         taxonService.update(t_abies_kawakamii_sensu_komarov);
1067
1068         commitAndStartNewTransaction(null);
1069
1070         pager = taxonService.findTaxaAndNamesByFullText(
1071                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1072                 "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1073         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1074
1075     }
1076
1077     @Test
1078     @DataSet
1079     @Ignore // remove once http://dev.e-taxonomy.eu/trac/ticket/5477 is solved
1080     public final void testFindTaxaAndNamesByFullText_AreaFilter_issue5477() throws IOException, LuceneParseException, LuceneMultiSearchException {
1081
1082         Set<NamedArea> a_germany_canada_russia = new HashSet<NamedArea>();
1083         a_germany_canada_russia.add(germany);
1084         a_germany_canada_russia.add(canada);
1085         a_germany_canada_russia.add(russia);
1086
1087
1088         Set<PresenceAbsenceTerm> absent = new HashSet<PresenceAbsenceTerm>();
1089         absent.add(PresenceAbsenceTerm.ABSENT());
1090
1091         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(UUID.fromString(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1092         Taxon t_abies_alba = (Taxon)taxonService.find(UUID.fromString(ABIES_ALBA_UUID));
1093         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1094
1095         /* Since the upgrade from hibernate search 4 to 5.5
1096          * triggering an update of t_abies_alba is no longer sufficient to also update the
1097          * document of t_abies_kawakamii_sensu_komarov in the lucene index.
1098          * the last test in testFindTaxaAndNamesByFullText_AreaFilter() failed in this case.
1099          * This situation is reproduced here:
1100          */
1101         taxonService.update(t_abies_alba);
1102
1103           commitAndStartNewTransaction(null);
1104
1105           Pager pager = taxonService.findTaxaAndNamesByFullText(
1106                   EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1107                   "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1108           Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1109     }
1110
1111
1112     /**
1113      * Regression test for #3119: fulltext search: Entity always null whatever search
1114      *
1115      * @throws IOException
1116      * @throws LuceneParseException
1117      * @throws LuceneMultiSearchException
1118      */
1119     @Test
1120     @DataSet
1121     public final void testFindByEverythingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1122
1123         refreshLuceneIndex();
1124
1125         Pager<SearchResult<TaxonBase>> pager;
1126
1127         // via Taxon
1128         pager = taxonService.findByEverythingFullText("Abies", null, null, true, null, null, null, null);
1129         logFreeTextSearchResults(pager, Level.DEBUG, null);
1130         Assert.assertTrue("Expecting at least 7 entities for 'Abies'", pager.getCount() > 7);
1131         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1132         Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1133
1134         // via DescriptionElement
1135         pager = taxonService.findByEverythingFullText("present", null, null, true, null, null, null, null);
1136         Assert.assertEquals("Expecting one entity when searching for area 'present'", 1, pager.getCount().intValue());
1137         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1138         Assert.assertEquals("Expecting Taxon entity", Taxon.class, CdmBase.deproxy(pager.getRecords().get(0).getEntity()).getClass());
1139         Assert.assertEquals("Expecting Taxon ", ABIES_BALSAMEA_UUID, pager.getRecords().get(0).getEntity().getUuid().toString());
1140
1141     }
1142
1143
1144     @Test
1145     @DataSet
1146     public final void findByEveryThingFullText() throws IOException, LuceneParseException, LuceneMultiSearchException {
1147
1148         refreshLuceneIndex();
1149
1150         Pager<SearchResult<TaxonBase>> pager;
1151
1152         pager = taxonService.findByEverythingFullText("genus", null, null,  false, null, null, null, null); // --> 1
1153         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1154
1155         //FIXME FAILS: abies balamea is returned twice, see also testFullText_Grouping()
1156         pager = taxonService.findByEverythingFullText("Balsam", null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1157         logFreeTextSearchResults(pager, Level.DEBUG, null);
1158         Assert.assertEquals("expecting to find the Abies balsamea via the GERMAN DescriptionElements", 1, pager.getCount().intValue());
1159
1160         pager = taxonService.findByEverythingFullText("Abies", null, null, true, null, null, null, null);
1161         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
1162         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
1163         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
1164         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
1165         Assert.assertTrue("first fragments should contains serch term", fragments[0].toLowerCase().contains("<b>abies</b>"));
1166     }
1167
1168 //    @SuppressWarnings("rawtypes")
1169 //    @Test
1170 //    @DataSet
1171 //    public final void benchmarkFindTaxaAndNamesHql() throws IOException, LuceneParseException {
1172 //
1173 //        createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1174 //
1175 //        IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1176 //        configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1177 //        configurator.setMatchMode(MatchMode.BEGINNING);
1178 //        configurator.setDoTaxa(false);
1179 //        configurator.setDoSynonyms(false);
1180 //        configurator.setDoNamesWithoutTaxa(false);
1181 //        configurator.setDoTaxaByCommonNames(true);
1182 //
1183 //        Pager<IdentifiableEntity> pager;
1184 //
1185 //        long startMillis = System.currentTimeMillis();
1186 //        for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1187 //            pager = taxonService.findTaxaAndNames(configurator);
1188 //            if (logger.isDebugEnabled()) {
1189 //                logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1190 //            }
1191 //        }
1192 //        double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1193 //        logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1194 //    }
1195
1196     @SuppressWarnings("rawtypes")
1197     @Test
1198     @DataSet
1199     public final void benchmarkFindByCommonNameHql() throws IOException, LuceneParseException {
1200
1201 //        printDataSet(System.err, new String[] { "TaxonBase" });
1202
1203         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1204
1205         IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl<>();
1206         configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1207         configurator.setMatchMode(MatchMode.BEGINNING);
1208         configurator.setDoTaxa(false);
1209         configurator.setDoSynonyms(false);
1210         configurator.setDoNamesWithoutTaxa(false);
1211         configurator.setDoTaxaByCommonNames(true);
1212
1213         Pager<IdentifiableEntity> pager;
1214
1215         long startMillis = System.currentTimeMillis();
1216         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1217             pager = taxonService.findTaxaAndNames(configurator);
1218             if (logger.isDebugEnabled()) {
1219                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1220             }
1221         }
1222         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1223         logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1224     }
1225
1226     @SuppressWarnings("rawtypes")
1227     @Test
1228     @DataSet
1229     public final void benchmarkFindByCommonNameLucene() throws IOException, LuceneParseException {
1230
1231         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1232
1233         refreshLuceneIndex();
1234
1235         Pager<SearchResult<TaxonBase>> pager;
1236
1237         long startMillis = System.currentTimeMillis();
1238         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1239             pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
1240             if (logger.isDebugEnabled()) {
1241                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getEntity().getTitleCache());
1242             }
1243         }
1244         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1245         logger.info("Benchmark result - [find taxon by CommonName via lucene] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1246     }
1247
1248     /**
1249      * uncomment @Test annotation to create the dataset for this test
1250      */
1251     @Override
1252     //    @Test
1253     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="BlankDataSet.xml")
1254     public final void createTestDataSet() throws FileNotFoundException {
1255
1256         Classification europeanAbiesClassification = Classification.NewInstance("European Abies");
1257         europeanAbiesClassification.setUuid(UUID.fromString(CLASSIFICATION_UUID));
1258         classificationService.save(europeanAbiesClassification);
1259
1260         Classification alternativeClassification = Classification.NewInstance("Abies alternative");
1261         alternativeClassification.setUuid(UUID.fromString(CLASSIFICATION_ALT_UUID));
1262         classificationService.save(alternativeClassification);
1263
1264         Reference sec = ReferenceFactory.newBook();
1265         sec.setTitleCache("Kohlbecker, A., Testcase standart views, 2013", true);
1266         Reference sec_sensu = ReferenceFactory.newBook();
1267         sec_sensu.setTitleCache("Komarov, V. L., Flora SSSR 29", true);
1268         referenceService.save(sec);
1269         referenceService.save(sec_sensu);
1270
1271         IBotanicalName n_abies = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
1272         n_abies.setNameCache("Abies", true);
1273         Taxon t_abies = Taxon.NewInstance(n_abies, sec);
1274         taxonService.save(t_abies);
1275
1276         IBotanicalName n_abies_alba = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1277         n_abies_alba.setNameCache("Abies alba", true);
1278         Taxon t_abies_alba = Taxon.NewInstance(n_abies_alba, sec);
1279         t_abies_alba.setUuid(UUID.fromString(ABIES_ALBA_UUID));
1280         taxonService.save(t_abies_alba);
1281
1282         IBotanicalName n_abies_subalpina = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1283         n_abies_subalpina.setNameCache("Abies subalpina", true);
1284         Synonym s_abies_subalpina = Synonym.NewInstance(n_abies_subalpina, sec);
1285         taxonService.save(s_abies_subalpina);
1286
1287         IBotanicalName n_abies_balsamea = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1288         n_abies_balsamea.setNameCache("Abies balsamea", true);
1289         Taxon t_abies_balsamea = Taxon.NewInstance(n_abies_balsamea, sec);
1290         t_abies_balsamea.setUuid(UUID.fromString(ABIES_BALSAMEA_UUID));
1291         t_abies_balsamea.addSynonym(s_abies_subalpina, SynonymType.SYNONYM_OF());
1292         taxonService.save(t_abies_balsamea);
1293
1294         IBotanicalName n_abies_grandis = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1295         n_abies_grandis.setNameCache("Abies grandis", true);
1296         Taxon t_abies_grandis = Taxon.NewInstance(n_abies_grandis, sec);
1297         taxonService.save(t_abies_grandis);
1298
1299         IBotanicalName n_abies_kawakamii = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1300         n_abies_kawakamii.setNameCache("Abies kawakamii", true);
1301         Taxon t_abies_kawakamii = Taxon.NewInstance(n_abies_kawakamii, sec);
1302         t_abies_kawakamii.getTitleCache();
1303         taxonService.save(t_abies_kawakamii);
1304
1305         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1306         Taxon t_abies_kawakamii_sensu_komarov = Taxon.NewInstance(n_abies_kawakamii, sec_sensu);
1307         taxonService.save(t_abies_kawakamii_sensu_komarov);
1308         t_abies_kawakamii_sensu_komarov.addTaxonRelation(t_abies_balsamea, TaxonRelationshipType.MISAPPLIED_NAME_FOR(), null, null);
1309         taxonService.saveOrUpdate(t_abies_kawakamii_sensu_komarov);
1310
1311         IBotanicalName n_abies_lasiocarpa = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1312         n_abies_lasiocarpa.setNameCache("Abies lasiocarpa", true);
1313         Taxon t_abies_lasiocarpa = Taxon.NewInstance(n_abies_lasiocarpa, sec);
1314         taxonService.save(t_abies_lasiocarpa);
1315
1316         // add taxa to classifications
1317         europeanAbiesClassification.addChildTaxon(t_abies_balsamea, null, null);
1318         alternativeClassification.addChildTaxon(t_abies_lasiocarpa, null, null);
1319         classificationService.saveOrUpdate(europeanAbiesClassification);
1320         classificationService.saveOrUpdate(alternativeClassification);
1321
1322         //
1323         // Description
1324         //
1325         TaxonDescription d_abies_alba = TaxonDescription.NewInstance(t_abies_alba);
1326         TaxonDescription d_abies_balsamea = TaxonDescription.NewInstance(t_abies_balsamea);
1327
1328         d_abies_alba.setUuid(UUID.fromString(D_ABIES_ALBA_UUID));
1329         d_abies_balsamea.setUuid(UUID.fromString(D_ABIES_BALSAMEA_UUID));
1330
1331
1332         // CommonTaxonName
1333         d_abies_alba.addElement(CommonTaxonName.NewInstance("Wei"+UTF8.SHARP_S+"tanne", Language.GERMAN()));
1334         d_abies_alba.addElement(CommonTaxonName.NewInstance("silver fir", Language.ENGLISH()));
1335         d_abies_alba.addElement(Distribution
1336                 .NewInstance(
1337                         germany,
1338                         PresenceAbsenceTerm.NATIVE()));
1339         d_abies_alba.addElement(Distribution
1340                 .NewInstance(
1341                         russia,
1342                         PresenceAbsenceTerm.ABSENT()));
1343
1344         // TextData
1345         d_abies_balsamea
1346             .addElement(TextData
1347                     .NewInstance(
1348                             "Die Balsam-Tanne (Abies balsamea) ist eine Pflanzenart aus der Gattung der Tannen (Abies). Sie wÃ¤chst im nordÃ¶stlichen Nordamerika, wo sie sowohl Tief- als auch Bergland besiedelt. Sie gilt als relativ anspruchslos gegenÃ¼ber dem Standort und ist frosthart. In vielen Teilen des natÃ¼rlichen Verbreitungsgebietes stellt sie die Klimaxbaumart dar.",
1349                             Language.GERMAN(), null));
1350         d_abies_balsamea
1351         .addElement(CommonTaxonName
1352                 .NewInstance(
1353                         "Balsam-Tanne",
1354                         Language.GERMAN(), null));
1355
1356         d_abies_balsamea
1357         .addElement(Distribution
1358                 .NewInstance(
1359                         canada,
1360                         PresenceAbsenceTerm.PRESENT()));
1361
1362         d_abies_balsamea
1363         .addElement(Distribution
1364                 .NewInstance(
1365                         germany,
1366                         PresenceAbsenceTerm.NATIVE()));
1367
1368         d_abies_balsamea
1369                 .addElement(TextData
1370                         .NewInstance(
1371                                 TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_LONG,
1372                                 Language.RUSSIAN(), null));
1373         d_abies_balsamea
1374         .addElement(CommonTaxonName
1375                 .NewInstance(
1376                         TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_SHORT,
1377                         Language.RUSSIAN(), null));
1378         descriptionService.saveOrUpdate(d_abies_balsamea);
1379
1380         setComplete();
1381         endTransaction();
1382
1383
1384         writeDbUnitDataSetFile(new String[] {
1385             "TAXONBASE", "TAXONNAME",
1386             "TAXONRELATIONSHIP",
1387             "REFERENCE", "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
1388             "AGENTBASE", "HOMOTYPICALGROUP",
1389             "CLASSIFICATION", "TAXONNODE",
1390             "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING",
1391             "HIBERNATE_SEQUENCES" // IMPORTANT!!!
1392             });
1393
1394     }
1395
1396     /**
1397      *
1398      */
1399     private void refreshLuceneIndex() {
1400
1401 //        commitAndStartNewTransaction(null);
1402         commit();
1403         endTransaction();
1404         indexer.purge(DefaultProgressMonitor.NewInstance());
1405         indexer.reindex(typesToIndex, DefaultProgressMonitor.NewInstance());
1406         startNewTransaction();
1407 //        commitAndStartNewTransaction(null);
1408     }
1409
1410     /**
1411      * @param numberOfNew
1412      *
1413      */
1414     private void createRandomTaxonWithCommonName(int numberOfNew) {
1415
1416         logger.debug(String.format("creating %1$s random taxan with CommonName", numberOfNew));
1417
1418         commitAndStartNewTransaction(null);
1419
1420         Reference sec = ReferenceFactory.newBook();
1421         referenceService.save(sec);
1422
1423         for (int i = numberOfNew; i < numberOfNew; i++) {
1424             RandomStringUtils.randomAlphabetic(10);
1425             String radomName = RandomStringUtils.randomAlphabetic(5) + " " + RandomStringUtils.randomAlphabetic(10);
1426             String radomCommonName = RandomStringUtils.randomAlphabetic(10);
1427
1428             IBotanicalName name = TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES());
1429             name.setNameCache(radomName, true);
1430             Taxon taxon = Taxon.NewInstance(name, sec);
1431             taxonService.save(taxon);
1432
1433             TaxonDescription description = TaxonDescription.NewInstance(taxon);
1434             description.addElement(CommonTaxonName.NewInstance(radomCommonName, Language.GERMAN()));
1435             descriptionService.save(description);
1436         }
1437
1438         commitAndStartNewTransaction(null);
1439     }
1440
1441     private <T extends CdmBase> void logFreeTextSearchResults(Pager<SearchResult<T>> pager, Level level, String[] docFields){
1442         if(level == null){
1443             level = Level.DEBUG;
1444         }
1445         if(logger.isEnabledFor(level)){
1446             StringBuilder b = new StringBuilder();
1447             b.append("\n");
1448             int i = 0;
1449             for(SearchResult sr : pager.getRecords()){
1450
1451                 b.append(" ").append(i++).append(" - ");
1452                 b.append("score:").append(sr.getScore()).append(", ");
1453
1454                 if(docFields != null){
1455                     b.append("docs : ");
1456                     for(Document doc : sr.getDocs()) {
1457                         b.append("<");
1458                         for(String f : docFields){
1459                             b.append(f).append(":").append(Arrays.toString(doc.getValues(f)));
1460                         }
1461                         b.append(">");
1462                     }
1463                 }
1464
1465                 CdmBase entity = sr.getEntity();
1466                 if(entity == null){
1467                     b.append("NULL");
1468                 } else {
1469                     b.append(entity.getClass().getSimpleName()).
1470                         append(" [").append(entity.getId()).
1471                         append(" | ").append(entity.getUuid()).append("] : ").
1472                         append(entity.toString());
1473
1474                 }
1475                 b.append("\n");
1476             }
1477             logger.log(level, b);
1478         }
1479     }
1480
1481 }