cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/TaxonServiceSearchTest.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import static org.junit.Assert.assertEquals;
  13 import static org.junit.Assert.assertNotNull;
  14
  15 import java.io.FileNotFoundException;
  16 import java.io.IOException;
  17 import java.util.ArrayList;
  18 import java.util.Arrays;
  19 import java.util.EnumSet;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24 import java.util.UUID;
  25
  26 import org.apache.commons.lang.RandomStringUtils;
  27 import org.apache.log4j.Level;
  28 import org.apache.log4j.Logger;
  29 import org.apache.lucene.document.Document;
  30 import org.apache.lucene.index.CorruptIndexException;
  31 import org.apache.lucene.queryParser.ParseException;
  32 import org.junit.Assert;
  33 import org.junit.Before;
  34 import org.junit.Ignore;
  35 import org.junit.Test;
  36 import org.unitils.dbunit.annotation.DataSet;
  37 import org.unitils.spring.annotation.SpringBeanByType;
  38
  39 import eu.etaxonomy.cdm.api.service.config.FindTaxaAndNamesConfiguratorImpl;
  40 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  41 import eu.etaxonomy.cdm.api.service.pager.Pager;
  42 import eu.etaxonomy.cdm.api.service.search.ICdmMassIndexer;
  43 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  44 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  45 import eu.etaxonomy.cdm.common.UTF8;
  46 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  47 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  48 import eu.etaxonomy.cdm.model.common.CdmBase;
  49 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  50 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  51 import eu.etaxonomy.cdm.model.common.Language;
  52 import eu.etaxonomy.cdm.model.description.CategoricalData;
  53 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  54 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  55 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  56 import eu.etaxonomy.cdm.model.description.Distribution;
  57 import eu.etaxonomy.cdm.model.description.Feature;
  58 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  59 import eu.etaxonomy.cdm.model.description.State;
  60 import eu.etaxonomy.cdm.model.description.StateData;
  61 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  62 import eu.etaxonomy.cdm.model.description.TextData;
  63 import eu.etaxonomy.cdm.model.location.Country;
  64 import eu.etaxonomy.cdm.model.location.NamedArea;
  65 import eu.etaxonomy.cdm.model.name.BotanicalName;
  66 import eu.etaxonomy.cdm.model.name.NonViralName;
  67 import eu.etaxonomy.cdm.model.name.Rank;
  68 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  69 import eu.etaxonomy.cdm.model.reference.Reference;
  70 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  71 import eu.etaxonomy.cdm.model.taxon.Classification;
  72 import eu.etaxonomy.cdm.model.taxon.Synonym;
  73 import eu.etaxonomy.cdm.model.taxon.SynonymRelationshipType;
  74 import eu.etaxonomy.cdm.model.taxon.Taxon;
  75 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  76 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  77 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  78 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  79 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  80 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  81 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  82 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  83 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  84
  85 /**
  86  * @author a.babadshanjan, a.kohlbecker
  87  * @created 04.02.2009
  88  */
  89 public class TaxonServiceSearchTest extends CdmTransactionalIntegrationTest {
  90
  91     private static final String ABIES_BALSAMEA_UUID = "f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8";
  92
  93     private static final String ABIES_ALBA_UUID = "7dbd5810-a3e5-44b6-b563-25152b8867f4";
  94
  95     private static final String CLASSIFICATION_UUID = "2a5ceebb-4830-4524-b330-78461bf8cb6b";
  96
  97     private static final String CLASSIFICATION_ALT_UUID = "d7c741e3-ae9e-4a7d-a566-9e3a7a0b51ce";
  98
  99     private static final String D_ABIES_BALSAMEA_UUID = "900108d8-e6ce-495e-b32e-7aad3099135e";
 100
 101     private static final String D_ABIES_ALBA_UUID = "ec8bba03-d993-4c85-8472-18b14942464b";
 102
 103     private static final String D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID = "e9d8c2fd-6409-46d5-9c2e-14a2bbb1b2b1";
 104     private static final int NUM_OF_NEW_RADOM_ENTITIES = 1000;
 105
 106     private static Logger logger = Logger.getLogger(TaxonServiceSearchTest.class);
 107
 108
 109
 110     @SpringBeanByType
 111     private ITaxonService taxonService;
 112     @SpringBeanByType
 113     private ITermService termService;
 114     @SpringBeanByType
 115     private IClassificationService classificationService;
 116     @SpringBeanByType
 117     private IReferenceService referenceService;
 118     @SpringBeanByType
 119     private IDescriptionService descriptionService;
 120     @SpringBeanByType
 121     private INameService nameService;
 122     @SpringBeanByType
 123     private ICdmMassIndexer indexer;
 124
 125     @SpringBeanByType
 126     private ITaxonNodeService nodeService;
 127
 128     private static final int BENCHMARK_ROUNDS = 300;
 129
 130     private Set<Class<? extends CdmBase>> typesToIndex = null;
 131
 132     private NamedArea germany;
 133     private NamedArea france ;
 134     private NamedArea russia ;
 135     private NamedArea canada ;
 136
 137     /**
 138      * @throws java.lang.Exception
 139      */
 140     @Before
 141     public void setUp() throws Exception {
 142         typesToIndex = new HashSet<Class<? extends CdmBase>>();
 143         typesToIndex.add(DescriptionElementBase.class);
 144         typesToIndex.add(TaxonBase.class);
 145         typesToIndex.add(TaxonRelationship.class);
 146
 147         germany =  Country.GERMANY();
 148         france = Country.FRANCEFRENCHREPUBLIC();
 149         russia = Country.RUSSIANFEDERATION();
 150         canada = Country.CANADA();
 151
 152
 153     }
 154
 155     @Test
 156     public void testDbUnitUsageTest() throws Exception {
 157         assertNotNull("taxonService should exist", taxonService);
 158         assertNotNull("nameService should exist", nameService);
 159     }
 160
 161     /**
 162      * Test method for
 163      * {@link eu.etaxonomy.cdm.api.service.TaxonServiceImpl#findTaxaAndNames(eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator)}
 164      * .
 165      */
 166     @Test
 167     @DataSet
 168     public final void testFindTaxaAndNames() {
 169
 170         // pass 1
 171         IFindTaxaAndNamesConfigurator<?> configurator = new FindTaxaAndNamesConfiguratorImpl();
 172         configurator.setTitleSearchString("Abies*");
 173         configurator.setMatchMode(MatchMode.BEGINNING);
 174         configurator.setDoTaxa(true);
 175         configurator.setDoSynonyms(true);
 176         configurator.setDoNamesWithoutTaxa(true);
 177         configurator.setDoTaxaByCommonNames(true);
 178
 179         Pager<IdentifiableEntity> pager = taxonService.findTaxaAndNames(configurator);
 180         List<IdentifiableEntity> list = pager.getRecords();
 181
 182         if (logger.isDebugEnabled()) {
 183             for (int i = 0; i < list.size(); i++) {
 184                 String nameCache = "";
 185                 if (list.get(i) instanceof NonViralName) {
 186                     nameCache = ((NonViralName<?>) list.get(i)).getNameCache();
 187                 } else if (list.get(i) instanceof TaxonBase) {
 188                     TaxonNameBase<?,?> taxonNameBase = ((TaxonBase) list.get(i)).getName();
 189                     nameCache = HibernateProxyHelper.deproxy(taxonNameBase, NonViralName.class).getNameCache();
 190                 } else {
 191                 }
 192                 logger.debug(list.get(i).getClass() + "(" + i + ")" + ": Name Cache = " + nameCache + ", Title Cache = "
 193                         + list.get(i).getTitleCache());
 194             }
 195         }
 196
 197         logger.debug("number of taxa: " + list.size());
 198         assertEquals(10, list.size());
 199         configurator.setTitleSearchString("Balsam-Tanne");
 200         pager = taxonService.findTaxaAndNames(configurator);
 201         list = pager.getRecords();
 202         assertEquals(1, list.size());
 203         // pass 2
 204 //        configurator.setDoTaxaByCommonNames(false);
 205 //        configurator.setDoMisappliedNames(true);
 206 //        configurator.setClassification(classificationService.load(UUID.fromString(CLASSIFICATION_UUID)));
 207 //        pager = taxonService.findTaxaAndNames(configurator);
 208 //        list = pager.getRecords();
 209 //        assertEquals(0, list.size());
 210
 211     }
 212
 213     /**
 214      * Test method for
 215      * {@link eu.etaxonomy.cdm.api.service.TaxonServiceImpl#searchTaxaByName(java.lang.String, eu.etaxonomy.cdm.model.reference.Reference)}
 216      * .
 217      */
 218     @Test
 219     @DataSet
 220     public final void testSearchTaxaByName() {
 221          IFindTaxaAndNamesConfigurator<?> configurator = new FindTaxaAndNamesConfiguratorImpl();
 222          configurator.setTitleSearchString("Abies bor*");
 223          configurator.setMatchMode(MatchMode.BEGINNING);
 224          configurator.setDoTaxa(true);
 225          configurator.setDoSynonyms(false);
 226          configurator.setDoNamesWithoutTaxa(true);
 227          configurator.setDoTaxaByCommonNames(false);
 228
 229         List<UuidAndTitleCache<IdentifiableEntity>> list = taxonService.findTaxaAndNamesForEditor(configurator);
 230
 231          Assert.assertEquals("Expecting one entity", 1, list.size());
 232
 233          configurator.setTitleSearchString("silver fir");
 234          configurator.setMatchMode(MatchMode.BEGINNING);
 235          configurator.setDoTaxa(false);
 236          configurator.setDoSynonyms(false);
 237          configurator.setDoNamesWithoutTaxa(true);
 238          configurator.setDoTaxaByCommonNames(true);
 239
 240          list = taxonService.findTaxaAndNamesForEditor(configurator);
 241
 242          Assert.assertEquals("Expecting one entity", 1, list.size());
 243
 244     }
 245
 246     @SuppressWarnings("rawtypes")
 247     @Test
 248     @DataSet
 249     public final void testPurgeIndex() throws CorruptIndexException, IOException, ParseException {
 250
 251         refreshLuceneIndex();
 252
 253         Pager<SearchResult<TaxonBase>> pager;
 254
 255         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 7
 256         Assert.assertEquals("Expecting 8 entities", Long.valueOf(8), pager.getCount());
 257
 258         indexer.purge(null);
 259         commitAndStartNewTransaction(null);
 260
 261         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 0
 262         Assert.assertEquals("Expecting no entities since the index has been purged", Long.valueOf(0), pager.getCount());
 263     }
 264
 265     @SuppressWarnings("rawtypes")
 266     @Test
 267     @DataSet
 268     public final void testFindByDescriptionElementFullText_CommonName() throws CorruptIndexException, IOException,
 269             ParseException {
 270
 271         refreshLuceneIndex();
 272
 273         Pager<SearchResult<TaxonBase>> pager;
 274
 275         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null, null,
 276                 false, null, null, null, null);
 277         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", Long.valueOf(1),
 278                 pager.getCount());
 279
 280         // the description containing the Nulltanne has no taxon attached,
 281         // taxon.id = null
 282         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Nulltanne", null, null, null,
 283                 false, null, null, null, null);
 284         Assert.assertEquals("Expecting no entity when searching for 'Nulltanne' ", Long.valueOf(0), pager.getCount());
 285
 286         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 287                 Arrays.asList(new Language[] { Language.GERMAN() }), false, null, null, null, null);
 288         Assert.assertEquals("Expecting one entity when searching in German", Long.valueOf(1), pager.getCount());
 289
 290         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 291                 Arrays.asList(new Language[] { Language.RUSSIAN() }), false, null, null, null, null);
 292         Assert.assertEquals("Expecting no entity when searching in Russian", Long.valueOf(0), pager.getCount());
 293
 294     }
 295
 296     @SuppressWarnings("rawtypes")
 297     @Test
 298     @DataSet
 299     public final void testFindByDescriptionElementFullText_Distribution() throws CorruptIndexException, IOException, ParseException {
 300
 301         refreshLuceneIndex();
 302
 303         Pager<SearchResult<TaxonBase>> pager;
 304         // by Area
 305         pager = taxonService.findByDescriptionElementFullText(null, "Canada", null, null, null, false, null, null, null, null);
 306         Assert.assertEquals("Expecting one entity when searching for arae 'Canada'", Long.valueOf(1), pager.getCount());
 307         // by Status
 308         pager = taxonService.findByDescriptionElementFullText(null, "present", null, null, null, false, null, null, null, null);
 309         Assert.assertEquals("Expecting one entity when searching for status 'present'", Long.valueOf(1), pager.getCount());
 310     }
 311
 312     @SuppressWarnings("rawtypes")
 313     @Test
 314     @DataSet
 315     public final void testFindByDescriptionElementFullText_wildcard() throws CorruptIndexException, IOException, ParseException {
 316
 317         refreshLuceneIndex();
 318
 319         Pager<SearchResult<TaxonBase>> pager;
 320
 321         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
 322         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", Long.valueOf(1), pager.getCount());
 323     }
 324
 325     /**
 326      * Regression test for #3113 (hibernate search: wildcard query can cause BooleanQuery$TooManyClauses: maxClauseCount is set to 1024)
 327      *
 328      * @throws CorruptIndexException
 329      * @throws IOException
 330      * @throws ParseException
 331      */
 332     @SuppressWarnings("rawtypes")
 333     @Test
 334     @DataSet
 335     public final void testFindByDescriptionElementFullText_TooManyClauses() throws CorruptIndexException, IOException, ParseException {
 336
 337         // generate 1024 terms to reproduce the bug
 338         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 339         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 340         while(uniqueRandomStrs.size() < 1024){
 341             uniqueRandomStrs.add(RandomStringUtils.random(10, true, false));
 342         }
 343         for(String rndStr: uniqueRandomStrs){
 344             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 345         }
 346         descriptionService.saveOrUpdate(description);
 347         commitAndStartNewTransaction(null);
 348
 349         refreshLuceneIndex();
 350
 351         Pager<SearchResult<TaxonBase>> pager;
 352
 353         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, null, null, null, null);
 354         Assert.assertEquals("Expecting all 1024 entities grouped into one SearchResult item when searching for Rot*", 1, pager.getCount().intValue());
 355     }
 356
 357     /**
 358      * Regression test for #3116 (fulltext search: always only one page of results)
 359      *
 360      * @throws CorruptIndexException
 361      * @throws IOException
 362      * @throws ParseException
 363      */
 364     @SuppressWarnings("rawtypes")
 365     @Test
 366     @DataSet
 367     public final void testFullText_Paging() throws CorruptIndexException, IOException, ParseException {
 368
 369         Reference sec = ReferenceFactory.newDatabase();
 370         referenceService.save(sec);
 371
 372         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 373         int numOfItems = 100;
 374         while(uniqueRandomStrs.size() < numOfItems){
 375             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 376         }
 377
 378         for(String rndStr: uniqueRandomStrs){
 379
 380             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(Rank.SERIES()), sec);
 381             taxon.setTitleCache("Tax" + rndStr, true);
 382             taxonService.save(taxon);
 383
 384             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 385             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 386             descriptionService.saveOrUpdate(description);
 387         }
 388
 389         commitAndStartNewTransaction(new String[]{"TAXONBASE"});
 390         refreshLuceneIndex();
 391
 392         int pageSize = 10;
 393
 394         Pager<SearchResult<TaxonBase>> pager;
 395
 396         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, null, null, null);
 397         Assert.assertEquals("unexpeted number of pages", Integer.valueOf(numOfItems / pageSize), pager.getPagesAvailable());
 398         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 9, null, null);
 399         Assert.assertNotNull("last page must have records", pager.getRecords());
 400         Assert.assertNotNull("last item on last page must exist", pager.getRecords().get(0));
 401         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 10, null, null);
 402         Assert.assertNotNull("last page + 1 must not have any records", pager.getRecords());
 403     }
 404
 405     /**
 406      * test for max score and sort by score of hit groups
 407      * with all matches per taxon in a single TextData  element
 408      * see {@link #testFullText_ScoreAndOrder_2()} for the complement
 409      * test with matches in multiple TextData per taxon
 410      *
 411      * @throws CorruptIndexException
 412      * @throws IOException
 413      * @throws ParseException
 414      */
 415     @SuppressWarnings("rawtypes")
 416     @Test
 417     @DataSet
 418     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 419     public final void testFullText_ScoreAndOrder_1() throws CorruptIndexException, IOException, ParseException {
 420
 421         int numOfTaxa = 3;
 422
 423         UUID[] taxonUuids = new UUID[numOfTaxa];
 424         StringBuilder text = new StringBuilder();
 425
 426         for(int i = 0; i < numOfTaxa; i++){
 427
 428             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(null), null);
 429             taxon.setTitleCache("Taxon_" + i, true);
 430             taxonUuids[i] = taxon.getUuid();
 431             taxonService.save(taxon);
 432
 433             text.append(" ").append("Rot");
 434             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 435             description.addElement(TextData.NewInstance(text.toString(), Language.DEFAULT(), null));
 436             descriptionService.saveOrUpdate(description);
 437         }
 438
 439         commitAndStartNewTransaction(null);
 440         refreshLuceneIndex();
 441
 442         Pager<SearchResult<TaxonBase>> pager;
 443
 444         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 445         for(int i = 0; i < numOfTaxa; i++){
 446             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 447         }
 448         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 449     }
 450
 451     /**
 452      * test for max score and sort by score of hit groups
 453      * with all matches per taxon in a multiple TextData elements
 454      * see {@link #testFullText_ScoreAndOrder_1()} for the complement
 455      * test with matches in a single TextData per taxon
 456      *
 457      * @throws CorruptIndexException
 458      * @throws IOException
 459      * @throws ParseException
 460      */
 461     @SuppressWarnings("rawtypes")
 462     @Test
 463     @DataSet
 464     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 465     public final void testFullText_ScoreAndOrder_2() throws CorruptIndexException, IOException, ParseException {
 466
 467         int numOfTaxa = 3;
 468
 469         UUID[] taxonUuids = new UUID[numOfTaxa];
 470
 471         for(int i = 0; i < numOfTaxa; i++){
 472
 473             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(null), null);
 474             taxon.setTitleCache("Taxon_" + i, true);
 475             taxonUuids[i] = taxon.getUuid();
 476             taxonService.save(taxon);
 477
 478             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 479             for(int k = 0; k < i; k++){
 480                 description.addElement(TextData.NewInstance("Rot", Language.DEFAULT(), null));
 481             }
 482             descriptionService.saveOrUpdate(description);
 483         }
 484
 485         commitAndStartNewTransaction(null);
 486         refreshLuceneIndex();
 487
 488         Pager<SearchResult<TaxonBase>> pager;
 489
 490         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 491         for(int i = 0; i < numOfTaxa; i++){
 492             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 493         }
 494         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 495     }
 496
 497
 498     /**
 499      * @throws CorruptIndexException
 500      * @throws IOException
 501      * @throws ParseException
 502      * @throws LuceneMultiSearchException
 503      */
 504     @Test
 505     @DataSet
 506     public final void testFullText_Grouping() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 507
 508         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 509         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 510         int numOfItems = 100;
 511         while(uniqueRandomStrs.size() < numOfItems){
 512             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 513         }
 514         for(String rndStr: uniqueRandomStrs){
 515             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 516         }
 517         descriptionService.saveOrUpdate(description);
 518
 519         commitAndStartNewTransaction(new String[]{"DESCRIPTIONELEMENTBASE"});
 520
 521         refreshLuceneIndex();
 522
 523         int pageSize = 10;
 524
 525         Pager<SearchResult<TaxonBase>> pager;
 526         boolean highlightFragments = true;
 527
 528         // test with findByDescriptionElementFullText
 529         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, highlightFragments, pageSize, null, null, null);
 530         logSearchResults(pager, Level.DEBUG, null);
 531         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 532         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 533         Map<String, String[]> highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 534         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 535         int maxDocsPerGroup = 10;
 536         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 537
 538         // test with findByEverythingFullText
 539         pager = taxonService.findByEverythingFullText( "Rot*", null, null, highlightFragments, pageSize, null, null, null);
 540         logSearchResults(pager, Level.DEBUG, null);
 541         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 542         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 543         highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 544         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 545         maxDocsPerGroup = 10;
 546         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 547
 548     }
 549
 550     @SuppressWarnings("rawtypes")
 551     @Test
 552     @DataSet
 553     @Ignore
 554     public final void testFindByDescriptionElementFullText_TextData() throws CorruptIndexException, IOException, ParseException {
 555
 556         refreshLuceneIndex();
 557
 558         Pager<SearchResult<TaxonBase>> pager;
 559         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, false, null, null, null, null);
 560         logSearchResults(pager, Level.DEBUG, null);
 561         Assert.assertEquals("Expecting one entity when searching for any TextData", Long.valueOf(1), pager.getCount());
 562         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 563         Assert.assertTrue("Expecting two docs, one for RUSSIAN and one for GERMAN", pager.getRecords().get(0).getDocs().size() == 2);
 564         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 565
 566
 567         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, null, null, false, null, null, null, null);
 568         Assert.assertEquals("Expecting one entity when searching for any type", Long.valueOf(1), pager.getCount());
 569
 570         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.UNKNOWN()}), null, false, null, null, null, null);
 571         Assert.assertEquals("Expecting one entity when searching for any type and for Feature DESCRIPTION", Long.valueOf(1), pager.getCount());
 572
 573         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER()}), null, false, null, null, null, null);
 574         Assert.assertEquals("Expecting no entity when searching for any type and for Feature CHROMOSOME_NUMBER", Long.valueOf(0), pager.getCount());
 575
 576         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER(), Feature.UNKNOWN()}), null, false, null, null, null, null);
 577         Assert.assertEquals("Expecting no entity when searching for any type and for Feature DESCRIPTION or CHROMOSOME_NUMBER", Long.valueOf(1), pager.getCount());
 578
 579         pager = taxonService.findByDescriptionElementFullText(Distribution.class, "Abies", null, null, null, false, null, null, null, null);
 580         Assert.assertEquals("Expecting no entity when searching for Distribution", Long.valueOf(0), pager.getCount());
 581
 582         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{}), false, null, null, null, null);
 583         Assert.assertEquals("Expecting one entity", Long.valueOf(1), pager.getCount());
 584         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 585
 586         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.RUSSIAN()}), false, null, null, null, null);
 587         Assert.assertEquals("Expecting one entity", Long.valueOf(1), pager.getCount());
 588         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 589
 590         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 591         Assert.assertEquals("Expecting no entity", Long.valueOf(0), pager.getCount());
 592
 593         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 594         Assert.assertEquals("Expecting one entity", Long.valueOf(1), pager.getCount());
 595         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 596     }
 597
 598     @SuppressWarnings("rawtypes")
 599     @Test
 600     @DataSet
 601     public final void testFindByDescriptionElementFullText_MultipleWords() throws CorruptIndexException, IOException, ParseException {
 602
 603         refreshLuceneIndex();
 604
 605         // Pflanzenart aus der Gattung der Tannen
 606
 607         Pager<SearchResult<TaxonBase>> pager;
 608         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, false, null, null, null, null);
 609         Assert.assertEquals("OR search : Expecting one entity", Long.valueOf(1), pager.getCount());
 610
 611         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Wespen", null, null, null, false, null, null, null, null);
 612         Assert.assertEquals("OR search : Expecting one entity", Long.valueOf(1), pager.getCount());
 613
 614         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, false, null, null, null, null);
 615         Assert.assertEquals("AND search : Expecting one entity", Long.valueOf(1), pager.getCount());
 616
 617         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Wespen", null, null, null, false, null, null, null, null);
 618         Assert.assertEquals("AND search : Expecting no entity", Long.valueOf(0), pager.getCount());
 619
 620         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, false, null, null, null, null);
 621         Assert.assertEquals("Phrase search : Expecting one entity", Long.valueOf(1), pager.getCount());
 622
 623         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Wespen\"", null, null, null, false, null, null, null, null);
 624         Assert.assertEquals("Phrase search : Expecting one entity", Long.valueOf(0), pager.getCount());
 625
 626
 627     }
 628
 629
 630     @SuppressWarnings("rawtypes")
 631     @Test
 632     @DataSet
 633     public final void testFindByDescriptionElementFullText_modify_DescriptionElement() throws CorruptIndexException, IOException, ParseException {
 634
 635         refreshLuceneIndex();
 636
 637         Pager<SearchResult<TaxonBase>> pager;
 638         //
 639         // modify the DescriptionElement
 640         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 641         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 642         Document indexDocument = pager.getRecords().get(0).getDocs().iterator().next();
 643         String[] descriptionElementUuidStr = indexDocument.getValues("uuid");
 644         String[] inDescriptionUuidStr = indexDocument.getValues("inDescription.uuid");
 645         // is only one uuid!
 646         DescriptionElementBase textData = descriptionService.getDescriptionElementByUuid(UUID.fromString(descriptionElementUuidStr[0]));
 647
 648         ((TextData)textData).removeText(Language.GERMAN());
 649         ((TextData)textData).putText(Language.SPANISH_CASTILIAN(), "abeto bals"+UTF8.SMALL_A_ACUTE+"mico");
 650
 651         descriptionService.saveDescriptionElement(textData);
 652         commitAndStartNewTransaction(null);
 653 //        printDataSet(System.out, new String[] {
 654 //                "DESCRIPTIONELEMENTBASE", "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING" }
 655 //        );
 656
 657         //
 658         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 659         Assert.assertEquals("The german 'Balsam-Tanne' TextData should no longer be indexed", Long.valueOf(0), pager.getCount());
 660         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 661         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", Long.valueOf(1), pager.getCount());
 662         pager = taxonService.findByDescriptionElementFullText(TextData.class, "bals"+UTF8.SMALL_A_ACUTE+"mico", null, null, null, false, null, null, null, null);
 663         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", Long.valueOf(1), pager.getCount());
 664
 665         //
 666         // modify the DescriptionElement via the Description object
 667         DescriptionBase<?> description = descriptionService.find(UUID.fromString(inDescriptionUuidStr[0]));
 668         Set<DescriptionElementBase> elements = description.getElements();
 669         for( DescriptionElementBase elm : elements){
 670             if(elm.getUuid().toString().equals(descriptionElementUuidStr[0])){
 671                 ((TextData)elm).removeText(Language.SPANISH_CASTILIAN());
 672                 ((TextData)elm).putText(Language.POLISH(), "Jod"+UTF8.POLISH_L+"a balsamiczna");
 673             }
 674         }
 675         descriptionService.saveOrUpdate(description);
 676         commitAndStartNewTransaction(null);
 677         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 678         Assert.assertEquals("The spanish 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico' TextData should no longer be indexed", Long.valueOf(0), pager.getCount());
 679         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsamiczna", null, null, Arrays.asList(new Language[]{Language.POLISH()}), false, null, null, null, null);
 680         Assert.assertEquals("expecting to find the POLISH 'Jod"+UTF8.POLISH_L+"a balsamiczna'", Long.valueOf(1), pager.getCount());
 681     }
 682
 683     @SuppressWarnings("rawtypes")
 684     @Test
 685     @DataSet
 686     public final void testFindByDescriptionElementFullText_modify_Taxon() throws CorruptIndexException, IOException, ParseException {
 687
 688         refreshLuceneIndex();
 689
 690         Pager<SearchResult<TaxonBase>> pager;
 691         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
 692         TaxonDescription d_abies_balsamea = (TaxonDescription)descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 693
 694         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 695         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne'", Long.valueOf(1), pager.getCount());
 696
 697         // exchange the Taxon with another one via the Taxon object
 698         // 1.) remove existing description:
 699         t_abies_balsamea.removeDescription(d_abies_balsamea);
 700
 701         taxonService.saveOrUpdate(t_abies_balsamea);
 702         commitAndStartNewTransaction(null);
 703
 704         t_abies_balsamea = (Taxon)taxonService.find(t_abies_balsamea.getUuid());
 705
 706         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 707         Assert.assertEquals("'Balsam-Tanne' should no longer be found", Long.valueOf(0), pager.getCount());
 708
 709         // 2.) create new description and add to taxon:
 710         TaxonDescription d_abies_balsamea_new = TaxonDescription.NewInstance();
 711         d_abies_balsamea_new
 712                 .addElement(TextData
 713                         .NewInstance(
 714                                 "Die Balsamtanne ist mit bis zu 30 m Höhe ein mittelgro"+UTF8.SHARP_S+"er Baum und kann bis zu 200 Jahre alt werden",
 715                                 Language.GERMAN(), null));
 716         t_abies_balsamea.addDescription(d_abies_balsamea_new);
 717         // set authorshipCache to null to avoid validation exception,
 718         // this is maybe not needed in future,  see ticket #3344
 719         BotanicalName abies_balsamea = HibernateProxyHelper.deproxy(t_abies_balsamea.getName(), BotanicalName.class);
 720         abies_balsamea.setAuthorshipCache(null);
 721         taxonService.saveOrUpdate(t_abies_balsamea);
 722         commitAndStartNewTransaction(null);
 723
 724         printDataSet(System.out, new String[] {
 725                 "DESCRIPTIONBASE"
 726         });
 727
 728         pager = taxonService.findByDescriptionElementFullText(TextData.class, "mittelgro"+UTF8.SHARP_S+"er Baum", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 729         Assert.assertEquals("the taxon should be found via the new Description", Long.valueOf(1), pager.getCount());
 730     }
 731
 732     @SuppressWarnings("rawtypes")
 733     @Test
 734     @DataSet
 735     public final void testFindByDescriptionElementFullText_modify_Classification() throws CorruptIndexException, IOException, ParseException {
 736
 737         refreshLuceneIndex();
 738
 739         Pager<SearchResult<TaxonBase>> pager;
 740
 741         // put taxon into other classification, new taxon node
 742         Classification classification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 743         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 744
 745         // TODO: why is the test failing when the childNode is already retrieved here, and not after the following four lines?
 746         //TaxonNode childNode = classification.getChildNodes().iterator().next();
 747
 748         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 749         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne' even if filtering by classification", Long.valueOf(1), pager.getCount());
 750         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 751         Assert.assertEquals("GERMAN 'Balsam-Tanne' should NOT be found in other classification", Long.valueOf(0), pager.getCount());
 752
 753         // check for the right taxon node
 754         TaxonNode childNode = classification.getChildNodes().iterator().next();
 755         Assert.assertEquals("expecting Abies balsamea sec.", childNode.getTaxon().getUuid().toString(), ABIES_BALSAMEA_UUID);
 756         Assert.assertEquals("expecting default classification", childNode.getClassification().getUuid().toString(), CLASSIFICATION_UUID);
 757
 758         // moving the taxon around, the rootnode is only a proxy
 759         alternateClassification.setRootNode(HibernateProxyHelper.deproxy(alternateClassification.getRootNode(), TaxonNode.class));
 760         alternateClassification.addChildNode(childNode, null, null);
 761
 762         classificationService.saveOrUpdate(alternateClassification);
 763         commitAndStartNewTransaction(null);
 764
 765 //        printDataSet(System.out, new String[] {
 766 //            "TAXONBASE", "TAXONNODE", "CLASSIFICATION"
 767 //        });
 768
 769         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 770         Assert.assertEquals("GERMAN 'Balsam-Tanne' should now be found in other classification", Long.valueOf(1), pager.getCount());
 771
 772         classification.getChildNodes().clear();
 773         classificationService.saveOrUpdate(classification);
 774         commitAndStartNewTransaction(null);
 775
 776         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", classification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 777         Assert.assertEquals("Now the GERMAN 'Balsam-Tanne' should NOT be found in original classification", Long.valueOf(0), pager.getCount());
 778
 779     }
 780
 781     @SuppressWarnings("rawtypes")
 782     @Test
 783     @DataSet
 784     public final void testFindByDescriptionElementFullText_CategoricalData() throws CorruptIndexException, IOException, ParseException {
 785
 786         // add CategoricalData
 787         DescriptionBase d_abies_balsamea = descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 788         // Categorical data
 789         CategoricalData cdata = CategoricalData.NewInstance();
 790         cdata.setFeature(Feature.DESCRIPTION());
 791         State state = State.NewInstance("green", "green", "gn");
 792
 793         StateData statedata = StateData.NewInstance(state);
 794         statedata.putModifyingText(Language.ENGLISH(), "always, even during winter");
 795         cdata.addStateData(statedata);
 796         d_abies_balsamea.addElement(cdata);
 797
 798         UUID termUUID = termService.save(state).getUuid();
 799         descriptionService.save(d_abies_balsamea);
 800
 801         commitAndStartNewTransaction(null);
 802
 803 //        printDataSet(System.out, new String[] {
 804 //                 "STATEDATA", "STATEDATA_DEFINEDTERMBASE", "STATEDATA_LANGUAGESTRING", "LANGUAGESTRING"});
 805
 806         refreshLuceneIndex();
 807
 808         Pager<SearchResult<TaxonBase>> pager;
 809         pager = taxonService.findByDescriptionElementFullText(CategoricalData.class, "green", null, null, null, false, null, null, null, null);
 810         Assert.assertEquals("Expecting one entity", Long.valueOf(1), pager.getCount());
 811         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 812         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 813         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 814
 815
 816         //TODO modify the StateData
 817         TaxonBase taxon = pager.getRecords().get(0).getEntity();
 818
 819         String newName = "Quercus robur";
 820         taxon.setTitleCache(newName + " sec. ", true);
 821
 822         taxonService.saveOrUpdate(taxon);
 823         commitAndStartNewTransaction(null);
 824
 825         taxon = taxonService.find(taxon.getUuid());
 826         Assert.assertEquals(newName + " sec. ", taxon.getTitleCache());
 827         DefinedTermBase term = termService.find(termUUID);
 828
 829         termService.delete(term);
 830
 831     }
 832
 833     @SuppressWarnings("rawtypes")
 834     @Test
 835     @DataSet
 836     public final void testFindByDescriptionElementFullText_Highlighting() throws CorruptIndexException, IOException, ParseException {
 837
 838         refreshLuceneIndex();
 839
 840         Pager<SearchResult<TaxonBase>> pager;
 841         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, true, null, null, null, null);
 842         Assert.assertEquals("Expecting one entity when searching for any TextData", Long.valueOf(1), pager.getCount());
 843         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 844         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
 845         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 846         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Abies</B>"));
 847
 848         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, true, null, null, null, null);
 849         searchResult = pager.getRecords().get(0);
 850         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 851         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 852         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") || fragments[0].contains("<B>Tannen</B>"));
 853
 854         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, true, null, null, null, null);
 855         searchResult = pager.getRecords().get(0);
 856         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 857         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 858         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") && fragments[0].contains("<B>Tannen</B>"));
 859
 860         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, true, null, null, null, null);
 861         searchResult = pager.getRecords().get(0);
 862         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 863         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 864         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B> <B>aus</B> <B>der</B> <B>Gattung</B> <B>der</B> <B>Tannen</B>"));
 865
 866         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Gatt*", null, null, null, true, null, null, null, null);
 867         searchResult = pager.getRecords().get(0);
 868         Assert.assertTrue("Wildcard search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 869         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 870         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Gatt"));
 871     }
 872
 873
 874     @Test
 875     @DataSet
 876     public final void testFindByFullText() throws CorruptIndexException, IOException, ParseException {
 877
 878         refreshLuceneIndex();
 879
 880         Classification europeanAbiesClassification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 881
 882         Pager<SearchResult<TaxonBase>> pager;
 883
 884         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 7
 885         logSearchResults(pager, Level.DEBUG, null);
 886         Assert.assertEquals("Expecting 8 entities", Long.valueOf(8), pager.getCount());
 887
 888         pager = taxonService.findByFullText(Taxon.class, "Abies", null, null, true, null, null, null, null); // --> 6
 889         Assert.assertEquals("Expecting 7 entities", Long.valueOf(7), pager.getCount());
 890
 891         pager = taxonService.findByFullText(Synonym.class, "Abies", null, null, true, null, null, null, null); // --> 1
 892         Assert.assertEquals("Expecting 1 entity", Long.valueOf(1), pager.getCount());
 893
 894         pager = taxonService.findByFullText(TaxonBase.class, "sec", null, null, true, null, null, null, null); // --> 7
 895         Assert.assertEquals("Expecting 8 entities", Long.valueOf(8), pager.getCount());
 896
 897         pager = taxonService.findByFullText(null, "genus", null, null, true, null, null, null, null); // --> 1
 898         Assert.assertEquals("Expecting 1 entity", Long.valueOf(1), pager.getCount());
 899
 900         pager = taxonService.findByFullText(Taxon.class, "subalpina", null, null, true, null, null, null, null); // --> 0
 901         Assert.assertEquals("Expecting 0 entities", Long.valueOf(0), pager.getCount());
 902
 903         // synonym in classification ???
 904     }
 905
 906     @Test
 907     @DataSet
 908     public final void testPrepareByAreaSearch() throws IOException, ParseException {
 909
 910         List<PresenceAbsenceTerm> statusFilter = new ArrayList<PresenceAbsenceTerm>();
 911         List<NamedArea> areaFilter = new ArrayList<NamedArea>();
 912         areaFilter.add(germany);
 913         areaFilter.add(canada);
 914         areaFilter.add(russia);
 915
 916         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDistribution(areaFilter, statusFilter, null, 20, 0, null, null);
 917         Assert.assertEquals("Expecting 2 entities", Long.valueOf(2), Long.valueOf(pager.getRecords().size()));
 918
 919     }
 920
 921     @Test
 922     @DataSet
 923     public final void testFindTaxaAndNamesByFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 924
 925         refreshLuceneIndex();
 926
 927         Pager<SearchResult<TaxonBase>> pager;
 928
 929         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 930
 931
 932         pager = taxonService.findTaxaAndNamesByFullText(
 933                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 934                 "Abies", null, null, null, null, true, null, null, null, null);
 935 //        logPagerRecords(pager, Level.DEBUG);
 936         Assert.assertEquals("doTaxa & doSynonyms", Long.valueOf(8), pager.getCount());
 937
 938         pager = taxonService.findTaxaAndNamesByFullText(
 939                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 940                 "Abies", null, null, null, null, true, null, null, null, null);
 941 //        logPagerRecords(pager, Level.DEBUG);
 942         Assert.assertEquals("all search modes", Long.valueOf(8), pager.getCount());
 943
 944         pager = taxonService.findTaxaAndNamesByFullText(
 945                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 946                 "Abies", alternateClassification, null, null, null, true, null, null, null, null);
 947 //        logPagerRecords(pager, Level.DEBUG);
 948         Assert.assertEquals("all search modes, filtered by alternateClassification", Long.valueOf(1), pager.getCount());
 949
 950         pager = taxonService.findTaxaAndNamesByFullText(
 951                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
 952                 "Abies", null, null, null, null, true, null, null, null, null);
 953         Assert.assertEquals("Expecting 1 entity", Long.valueOf(1), pager.getCount());
 954         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 955         Assert.assertEquals(Synonym.class, searchResult.getEntity().getClass());
 956         // Abies subalpina sec. Kohlbecker, A., Testcase standart views, 2013
 957
 958
 959         pager = taxonService.findTaxaAndNamesByFullText(
 960                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 961                 "Abies", null, null, null, null, true, null, null, null, null);
 962         Assert.assertEquals("Expecting 0 entity", Long.valueOf(0), pager.getCount());
 963
 964
 965         pager = taxonService.findTaxaAndNamesByFullText(
 966                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 967                 "Tanne", null, null, null, null, true, null, null, null, null);
 968         Assert.assertEquals("Expecting 1 entity", Long.valueOf(1), Long.valueOf(pager.getRecords().size()));
 969         Assert.assertEquals("Expecting 1 entity", Long.valueOf(1), pager.getCount());
 970
 971         pager = taxonService.findTaxaAndNamesByFullText(
 972                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
 973                 "kawakamii", null, null, null, null, true, null, null, null, null);
 974         logSearchResults(pager, Level.DEBUG, null);
 975         Assert.assertEquals("Expecting 1 entity", Long.valueOf(1), pager.getCount());
 976
 977     }
 978
 979     @Test
 980     @DataSet
 981     public final void testFindTaxaAndNamesByFullText_Sort() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 982
 983         refreshLuceneIndex();
 984
 985         Pager<SearchResult<TaxonBase>> pager;
 986
 987         List<OrderHint> orderHints = new ArrayList<OrderHint>();
 988
 989         String[] docFields2log = new String[]{"id"};
 990
 991         // SortById
 992         orderHints.addAll(OrderHint.ORDER_BY_ID);
 993         pager = taxonService.findTaxaAndNamesByFullText(
 994                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
 995                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 996 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 997         int lastId = -1;
 998         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 999             if(lastId != -1){
1000                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
1001             }
1002             lastId = rs.getEntity().getId();
1003         }
1004
1005         orderHints.addAll(OrderHint.ORDER_BY_ID);
1006         pager = taxonService.findTaxaAndNamesByFullText(
1007                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1008                 "Abies", null, null, null, null, true, null, null, orderHints, null);
1009 //        logSearchResults(pager, Level.DEBUG, docFields2log);
1010
1011         lastId = -1;
1012         for(SearchResult<TaxonBase> rs : pager.getRecords()){
1013             if(lastId != -1){
1014                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
1015             }
1016             lastId = rs.getEntity().getId();
1017         }
1018
1019         // Sortby NOMENCLATURAL_SORT_ORDER TODO make assertions !!!
1020         orderHints.clear();
1021         orderHints.addAll(OrderHint.NOMENCLATURAL_SORT_ORDER);
1022         pager = taxonService.findTaxaAndNamesByFullText(
1023                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1024                 "Abies", null, null, null, null, true, null, null, orderHints, null);
1025         logSearchResults(pager, Level.DEBUG, null);
1026
1027     }
1028
1029     @Test
1030     @DataSet
1031     public final void testFindTaxaAndNamesByFullText_AreaFilter() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1032
1033         refreshLuceneIndex();
1034
1035         Pager<SearchResult<TaxonBase>> pager;
1036
1037         Set<NamedArea> a_germany_canada_russia = new HashSet<NamedArea>();
1038         a_germany_canada_russia.add(germany);
1039         a_germany_canada_russia.add(canada);
1040         a_germany_canada_russia.add(russia);
1041
1042         Set<NamedArea> a_russia = new HashSet<NamedArea>();
1043         a_russia.add(russia);
1044
1045         Set<PresenceAbsenceTerm> present = new HashSet<PresenceAbsenceTerm>();
1046         present.add(PresenceAbsenceTerm.PRESENT());
1047
1048         Set<PresenceAbsenceTerm> present_native = new HashSet<PresenceAbsenceTerm>();
1049         present_native.add(PresenceAbsenceTerm.PRESENT());
1050         present_native.add(PresenceAbsenceTerm.NATIVE());
1051
1052         Set<PresenceAbsenceTerm> absent = new HashSet<PresenceAbsenceTerm>();
1053         absent.add(PresenceAbsenceTerm.ABSENT());
1054
1055         pager = taxonService.findTaxaAndNamesByFullText(
1056                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
1057                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1058         logSearchResults(pager, Level.DEBUG, null);
1059         Assert.assertEquals("taxa with matching area filter", Long.valueOf(2), pager.getCount());
1060
1061         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1062         pager = taxonService.findTaxaAndNamesByFullText(
1063                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
1064                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1065         Assert.assertEquals("synonyms with matching area filter", Long.valueOf(1), pager.getCount());
1066
1067         pager = taxonService.findTaxaAndNamesByFullText(
1068                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1069                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1070         logSearchResults(pager, Level.DEBUG, null);
1071         Assert.assertEquals("taxa and synonyms with matching area filter", Long.valueOf(3), pager.getCount());
1072
1073         pager = taxonService.findTaxaAndNamesByFullText(
1074                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1075                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1076         Assert.assertEquals("taxa and synonyms with matching area & status filter 1", Long.valueOf(3), pager.getCount());
1077
1078         pager = taxonService.findTaxaAndNamesByFullText(
1079                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1080                 "Abies", null, a_germany_canada_russia, present, null, true, null, null, null, null);
1081         Assert.assertEquals("taxa and synonyms with matching area & status filter 2", Long.valueOf(2), pager.getCount());
1082
1083         pager = taxonService.findTaxaAndNamesByFullText(
1084                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1085                 "Abies", null, a_russia, present, null, true, null, null, null, null);
1086         Assert.assertEquals("taxa and synonyms with non matching area & status filter", Long.valueOf(0), pager.getCount());
1087
1088         pager = taxonService.findTaxaAndNamesByFullText(
1089                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
1090                 "Tanne", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1091         Assert.assertEquals("ByCommonNames with area filter", Long.valueOf(1), pager.getCount());
1092
1093         // abies_kawakamii_sensu_komarov as misapplied name for t_abies_balsamea
1094         pager = taxonService.findTaxaAndNamesByFullText(
1095                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1096                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1097         Assert.assertEquals("misappliedNames with matching area & status filter", Long.valueOf(1), pager.getCount());
1098
1099
1100         // 1. remove existing taxon relation
1101         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
1102         Set<TaxonRelationship> relsTo = t_abies_balsamea.getRelationsToThisTaxon();
1103         Assert.assertEquals(Long.valueOf(1), Long.valueOf(relsTo.size()));
1104         TaxonRelationship taxonRelation = relsTo.iterator().next();
1105         t_abies_balsamea.removeTaxonRelation(taxonRelation);
1106         taxonService.saveOrUpdate(t_abies_balsamea);
1107         commitAndStartNewTransaction(null);
1108
1109         pager = taxonService.findTaxaAndNamesByFullText(
1110                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1111                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1112         Assert.assertEquals("misappliedNames with matching area & status filter, should match nothing now", Long.valueOf(0), pager.getCount());
1113
1114         // 2. now add abies_kawakamii_sensu_komarov as misapplied name for t_abies_alba and search for misapplications in russia: ABSENT
1115         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(UUID.fromString(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1116         Taxon t_abies_alba = (Taxon)taxonService.find(UUID.fromString(ABIES_ALBA_UUID));
1117         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1118         taxonService.update(t_abies_alba);
1119         commitAndStartNewTransaction(null);
1120
1121         pager = taxonService.findTaxaAndNamesByFullText(
1122                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1123                 "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1124         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", Long.valueOf(1), pager.getCount());
1125
1126     }
1127
1128     /**
1129      * Regression test for #3119: fulltext search: Entity always null whatever search
1130      *
1131      * @throws CorruptIndexException
1132      * @throws IOException
1133      * @throws ParseException
1134      * @throws LuceneMultiSearchException
1135      */
1136     @Test
1137     @DataSet
1138     public final void testFindByEverythingFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1139
1140         refreshLuceneIndex();
1141
1142         Pager<SearchResult<TaxonBase>> pager;
1143
1144         // via Taxon
1145         pager = taxonService.findByEverythingFullText("Abies", null, null, true, null, null, null, null);
1146         logSearchResults(pager, Level.DEBUG, null);
1147         Assert.assertTrue("Expecting at least 7 entities for 'Abies'", pager.getCount() > 7);
1148         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1149         Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1150
1151         // via DescriptionElement
1152         pager = taxonService.findByEverythingFullText("present", null, null, true, null, null, null, null);
1153         Assert.assertEquals("Expecting one entity when searching for area 'present'", Long.valueOf(1), pager.getCount());
1154         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1155         Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1156         Assert.assertEquals("Expecting Taxon ", ABIES_BALSAMEA_UUID, pager.getRecords().get(0).getEntity().getUuid().toString());
1157
1158     }
1159
1160
1161     @Test
1162     @DataSet
1163     public final void findByEveryThingFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1164
1165         refreshLuceneIndex();
1166
1167         Pager<SearchResult<TaxonBase>> pager;
1168
1169         pager = taxonService.findByEverythingFullText("genus", null, null,  false, null, null, null, null); // --> 1
1170         Assert.assertEquals("Expecting 1 entity", Long.valueOf(1), pager.getCount());
1171
1172         //FIXME FAILS: abies balamea is returned twice, see also testFullText_Grouping()
1173         pager = taxonService.findByEverythingFullText("Balsam-Tanne", null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1174         logSearchResults(pager, Level.DEBUG, null);
1175         Assert.assertEquals("expecting to find the Abies balsamea via the GERMAN DescriptionElements", Long.valueOf(1), pager.getCount());
1176
1177         pager = taxonService.findByEverythingFullText("Abies", null, null, true, null, null, null, null);
1178         Assert.assertEquals("Expecting 8 entities", Long.valueOf(8), pager.getCount());
1179         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
1180         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
1181         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
1182         Assert.assertTrue("first fragments should contains serch term", fragments[0].toLowerCase().contains("<b>abies</b>"));
1183     }
1184
1185 //    @SuppressWarnings("rawtypes")
1186 //    @Test
1187 //    @DataSet
1188 //    public final void benchmarkFindTaxaAndNamesHql() throws CorruptIndexException, IOException, ParseException {
1189 //
1190 //        createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1191 //
1192 //        IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1193 //        configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1194 //        configurator.setMatchMode(MatchMode.BEGINNING);
1195 //        configurator.setDoTaxa(false);
1196 //        configurator.setDoSynonyms(false);
1197 //        configurator.setDoNamesWithoutTaxa(false);
1198 //        configurator.setDoTaxaByCommonNames(true);
1199 //
1200 //        Pager<IdentifiableEntity> pager;
1201 //
1202 //        long startMillis = System.currentTimeMillis();
1203 //        for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1204 //            pager = taxonService.findTaxaAndNames(configurator);
1205 //            if (logger.isDebugEnabled()) {
1206 //                logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1207 //            }
1208 //        }
1209 //        double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1210 //        logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1211 //    }
1212
1213     @SuppressWarnings("rawtypes")
1214     @Test
1215     @DataSet
1216     public final void benchmarkFindByCommonNameHql() throws CorruptIndexException, IOException, ParseException {
1217
1218 //        printDataSet(System.err, new String[] { "TaxonBase" });
1219
1220         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1221
1222         IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1223         configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1224         configurator.setMatchMode(MatchMode.BEGINNING);
1225         configurator.setDoTaxa(false);
1226         configurator.setDoSynonyms(false);
1227         configurator.setDoNamesWithoutTaxa(false);
1228         configurator.setDoTaxaByCommonNames(true);
1229
1230         Pager<IdentifiableEntity> pager;
1231
1232         long startMillis = System.currentTimeMillis();
1233         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1234             pager = taxonService.findTaxaAndNames(configurator);
1235             if (logger.isDebugEnabled()) {
1236                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1237             }
1238         }
1239         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1240         logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1241     }
1242
1243     @SuppressWarnings("rawtypes")
1244     @Test
1245     @DataSet
1246     public final void benchmarkFindByCommonNameLucene() throws CorruptIndexException, IOException, ParseException {
1247
1248         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1249
1250         refreshLuceneIndex();
1251
1252         Pager<SearchResult<TaxonBase>> pager;
1253
1254         long startMillis = System.currentTimeMillis();
1255         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1256             pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
1257             if (logger.isDebugEnabled()) {
1258                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getEntity().getTitleCache());
1259             }
1260         }
1261         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1262         logger.info("Benchmark result - [find taxon by CommonName via lucene] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1263     }
1264
1265     /**
1266      * uncomment @Test annotation to create the dataset for this test
1267      */
1268     @Override
1269     //    @Test
1270     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="BlankDataSet.xml")
1271     public final void createTestDataSet() throws FileNotFoundException {
1272
1273         Classification europeanAbiesClassification = Classification.NewInstance("European Abies");
1274         europeanAbiesClassification.setUuid(UUID.fromString(CLASSIFICATION_UUID));
1275         classificationService.save(europeanAbiesClassification);
1276
1277         Classification alternativeClassification = Classification.NewInstance("Abies alternative");
1278         alternativeClassification.setUuid(UUID.fromString(CLASSIFICATION_ALT_UUID));
1279         classificationService.save(alternativeClassification);
1280
1281         Reference<?> sec = ReferenceFactory.newBook();
1282         sec.setTitleCache("Kohlbecker, A., Testcase standart views, 2013", true);
1283         Reference<?> sec_sensu = ReferenceFactory.newBook();
1284         sec_sensu.setTitleCache("Komarov, V. L., Flora SSSR 29", true);
1285         referenceService.save(sec);
1286         referenceService.save(sec_sensu);
1287
1288         BotanicalName n_abies = BotanicalName.NewInstance(Rank.GENUS());
1289         n_abies.setNameCache("Abies", true);
1290         Taxon t_abies = Taxon.NewInstance(n_abies, sec);
1291         taxonService.save(t_abies);
1292
1293         BotanicalName n_abies_alba = BotanicalName.NewInstance(Rank.SPECIES());
1294         n_abies_alba.setNameCache("Abies alba", true);
1295         Taxon t_abies_alba = Taxon.NewInstance(n_abies_alba, sec);
1296         t_abies_alba.setUuid(UUID.fromString(ABIES_ALBA_UUID));
1297         taxonService.save(t_abies_alba);
1298
1299         BotanicalName n_abies_subalpina = BotanicalName.NewInstance(Rank.SPECIES());
1300         n_abies_subalpina.setNameCache("Abies subalpina", true);
1301         Synonym s_abies_subalpina = Synonym.NewInstance(n_abies_subalpina, sec);
1302         taxonService.save(s_abies_subalpina);
1303
1304         BotanicalName n_abies_balsamea = BotanicalName.NewInstance(Rank.SPECIES());
1305         n_abies_balsamea.setNameCache("Abies balsamea", true);
1306         Taxon t_abies_balsamea = Taxon.NewInstance(n_abies_balsamea, sec);
1307         t_abies_balsamea.setUuid(UUID.fromString(ABIES_BALSAMEA_UUID));
1308         t_abies_balsamea.addSynonym(s_abies_subalpina, SynonymRelationshipType.SYNONYM_OF());
1309         taxonService.save(t_abies_balsamea);
1310
1311         BotanicalName n_abies_grandis = BotanicalName.NewInstance(Rank.SPECIES());
1312         n_abies_grandis.setNameCache("Abies grandis", true);
1313         Taxon t_abies_grandis = Taxon.NewInstance(n_abies_grandis, sec);
1314         taxonService.save(t_abies_grandis);
1315
1316         BotanicalName n_abies_kawakamii = BotanicalName.NewInstance(Rank.SPECIES());
1317         n_abies_kawakamii.setNameCache("Abies kawakamii", true);
1318         Taxon t_abies_kawakamii = Taxon.NewInstance(n_abies_kawakamii, sec);
1319         t_abies_kawakamii.getTitleCache();
1320         taxonService.save(t_abies_kawakamii);
1321
1322         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1323         Taxon t_abies_kawakamii_sensu_komarov = Taxon.NewInstance(n_abies_kawakamii, sec_sensu);
1324         taxonService.save(t_abies_kawakamii_sensu_komarov);
1325         t_abies_kawakamii_sensu_komarov.addTaxonRelation(t_abies_balsamea, TaxonRelationshipType.MISAPPLIED_NAME_FOR(), null, null);
1326         taxonService.saveOrUpdate(t_abies_kawakamii_sensu_komarov);
1327
1328         BotanicalName n_abies_lasiocarpa = BotanicalName.NewInstance(Rank.SPECIES());
1329         n_abies_lasiocarpa.setNameCache("Abies lasiocarpa", true);
1330         Taxon t_abies_lasiocarpa = Taxon.NewInstance(n_abies_lasiocarpa, sec);
1331         taxonService.save(t_abies_lasiocarpa);
1332
1333         // add taxa to classifications
1334         europeanAbiesClassification.addChildTaxon(t_abies_balsamea, null, null);
1335         alternativeClassification.addChildTaxon(t_abies_lasiocarpa, null, null);
1336         classificationService.saveOrUpdate(europeanAbiesClassification);
1337         classificationService.saveOrUpdate(alternativeClassification);
1338
1339         //
1340         // Description
1341         //
1342         TaxonDescription d_abies_alba = TaxonDescription.NewInstance(t_abies_alba);
1343         TaxonDescription d_abies_balsamea = TaxonDescription.NewInstance(t_abies_balsamea);
1344
1345         d_abies_alba.setUuid(UUID.fromString(D_ABIES_ALBA_UUID));
1346         d_abies_balsamea.setUuid(UUID.fromString(D_ABIES_BALSAMEA_UUID));
1347
1348
1349         // CommonTaxonName
1350         d_abies_alba.addElement(CommonTaxonName.NewInstance("Wei"+UTF8.SHARP_S+"tanne", Language.GERMAN()));
1351         d_abies_alba.addElement(CommonTaxonName.NewInstance("silver fir", Language.ENGLISH()));
1352         d_abies_alba.addElement(Distribution
1353                 .NewInstance(
1354                         germany,
1355                         PresenceAbsenceTerm.NATIVE()));
1356         d_abies_alba.addElement(Distribution
1357                 .NewInstance(
1358                         russia,
1359                         PresenceAbsenceTerm.ABSENT()));
1360
1361         // TextData
1362         d_abies_balsamea
1363             .addElement(TextData
1364                     .NewInstance(
1365                             "Die Balsam-Tanne (Abies balsamea) ist eine Pflanzenart aus der Gattung der Tannen (Abies). Sie wÃ¤chst im nordÃ¶stlichen Nordamerika, wo sie sowohl Tief- als auch Bergland besiedelt. Sie gilt als relativ anspruchslos gegenÃ¼ber dem Standort und ist frosthart. In vielen Teilen des natÃ¼rlichen Verbreitungsgebietes stellt sie die Klimaxbaumart dar.",
1366                             Language.GERMAN(), null));
1367         d_abies_balsamea
1368         .addElement(CommonTaxonName
1369                 .NewInstance(
1370                         "Balsam-Tanne",
1371                         Language.GERMAN(), null));
1372
1373         d_abies_balsamea
1374         .addElement(Distribution
1375                 .NewInstance(
1376                         canada,
1377                         PresenceAbsenceTerm.PRESENT()));
1378
1379         d_abies_balsamea
1380         .addElement(Distribution
1381                 .NewInstance(
1382                         germany,
1383                         PresenceAbsenceTerm.NATIVE()));
1384
1385         d_abies_balsamea
1386                 .addElement(TextData
1387                         .NewInstance(
1388                                         TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_LONG,
1389                                 Language.RUSSIAN(), null));
1390         d_abies_balsamea
1391         .addElement(CommonTaxonName
1392                 .NewInstance(
1393                                 TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_SHORT,
1394                         Language.RUSSIAN(), null));
1395         descriptionService.saveOrUpdate(d_abies_balsamea);
1396
1397         setComplete();
1398         endTransaction();
1399
1400
1401         writeDbUnitDataSetFile(new String[] {
1402             "TAXONBASE", "TAXONNAMEBASE",
1403             "SYNONYMRELATIONSHIP", "TAXONRELATIONSHIP",
1404             "REFERENCE", "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
1405             "AGENTBASE", "HOMOTYPICALGROUP",
1406             "CLASSIFICATION", "TAXONNODE",
1407             "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING",
1408             "HIBERNATE_SEQUENCES" // IMPORTANT!!!
1409             });
1410
1411     }
1412
1413     /**
1414      *
1415      */
1416     private void refreshLuceneIndex() {
1417
1418 //        commitAndStartNewTransaction(null);
1419         commit();
1420         endTransaction();
1421         indexer.purge(DefaultProgressMonitor.NewInstance());
1422         indexer.reindex(typesToIndex, DefaultProgressMonitor.NewInstance());
1423         startNewTransaction();
1424 //        commitAndStartNewTransaction(null);
1425     }
1426
1427     /**
1428      * @param numberOfNew
1429      *
1430      */
1431     private void createRandomTaxonWithCommonName(int numberOfNew) {
1432
1433         logger.debug(String.format("creating %1$s random taxan with CommonName", numberOfNew));
1434
1435         commitAndStartNewTransaction(null);
1436
1437         Reference sec = ReferenceFactory.newBook();
1438         referenceService.save(sec);
1439
1440         for (int i = numberOfNew; i < numberOfNew; i++) {
1441             RandomStringUtils.randomAlphabetic(10);
1442             String radomName = RandomStringUtils.randomAlphabetic(5) + " " + RandomStringUtils.randomAlphabetic(10);
1443             String radomCommonName = RandomStringUtils.randomAlphabetic(10);
1444
1445             BotanicalName name = BotanicalName.NewInstance(Rank.SPECIES());
1446             name.setNameCache(radomName, true);
1447             Taxon taxon = Taxon.NewInstance(name, sec);
1448             taxonService.save(taxon);
1449
1450             TaxonDescription description = TaxonDescription.NewInstance(taxon);
1451             description.addElement(CommonTaxonName.NewInstance(radomCommonName, Language.GERMAN()));
1452             descriptionService.save(description);
1453         }
1454
1455         commitAndStartNewTransaction(null);
1456     }
1457
1458     private <T extends CdmBase> void logSearchResults(Pager<SearchResult<T>> pager, Level level, String[] docFields){
1459         if(level == null){
1460             level = Level.DEBUG;
1461         }
1462         if(logger.isEnabledFor(level)){
1463             StringBuilder b = new StringBuilder();
1464             b.append("\n");
1465             int i = 0;
1466             for(SearchResult sr : pager.getRecords()){
1467
1468                 b.append(" ").append(i++).append(" - ");
1469                 b.append("score:").append(sr.getScore()).append(", ");
1470
1471                 if(docFields != null){
1472                     b.append("docs : ");
1473                     for(Document doc : sr.getDocs()) {
1474                         b.append("<");
1475                         for(String f : docFields){
1476                             b.append(f).append(":").append(Arrays.toString(doc.getValues(f)));
1477                         }
1478                         b.append(">");
1479                     }
1480                 }
1481
1482                 CdmBase entity = sr.getEntity();
1483                 if(entity == null){
1484                     b.append("NULL");
1485                 } else {
1486                     b.append(entity.getClass().getSimpleName()).
1487                         append(" [").append(entity.getId()).
1488                         append(" | ").append(entity.getUuid()).append("] : ").
1489                         append(entity.toString());
1490
1491                 }
1492                 b.append("\n");
1493             }
1494             logger.log(level, b);
1495         }
1496     }
1497
1498 }