cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/TaxonServiceSearchTest.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import static org.junit.Assert.assertNotNull;
  13
  14 import java.io.FileNotFoundException;
  15 import java.io.IOException;
  16 import java.util.ArrayList;
  17 import java.util.Arrays;
  18 import java.util.EnumSet;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.UUID;
  24
  25 import org.apache.commons.lang.RandomStringUtils;
  26 import org.apache.log4j.Level;
  27 import org.apache.log4j.Logger;
  28 import org.apache.lucene.document.Document;
  29 import org.apache.lucene.index.CorruptIndexException;
  30 import org.apache.lucene.queryparser.classic.ParseException;
  31 import org.junit.Assert;
  32 import org.junit.Before;
  33 import org.junit.Ignore;
  34 import org.junit.Test;
  35 import org.unitils.dbunit.annotation.DataSet;
  36 import org.unitils.spring.annotation.SpringBeanByType;
  37
  38 import eu.etaxonomy.cdm.api.service.config.FindTaxaAndNamesConfiguratorImpl;
  39 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  40 import eu.etaxonomy.cdm.api.service.pager.Pager;
  41 import eu.etaxonomy.cdm.api.service.search.ICdmMassIndexer;
  42 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  43 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  44 import eu.etaxonomy.cdm.common.UTF8;
  45 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  46 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  47 import eu.etaxonomy.cdm.model.common.CdmBase;
  48 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  49 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  50 import eu.etaxonomy.cdm.model.common.Language;
  51 import eu.etaxonomy.cdm.model.description.CategoricalData;
  52 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  53 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  54 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  55 import eu.etaxonomy.cdm.model.description.Distribution;
  56 import eu.etaxonomy.cdm.model.description.Feature;
  57 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
  58 import eu.etaxonomy.cdm.model.description.State;
  59 import eu.etaxonomy.cdm.model.description.StateData;
  60 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  61 import eu.etaxonomy.cdm.model.description.TextData;
  62 import eu.etaxonomy.cdm.model.location.Country;
  63 import eu.etaxonomy.cdm.model.location.NamedArea;
  64 import eu.etaxonomy.cdm.model.name.BotanicalName;
  65 import eu.etaxonomy.cdm.model.name.Rank;
  66 import eu.etaxonomy.cdm.model.reference.Reference;
  67 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  68 import eu.etaxonomy.cdm.model.taxon.Classification;
  69 import eu.etaxonomy.cdm.model.taxon.Synonym;
  70 import eu.etaxonomy.cdm.model.taxon.SynonymType;
  71 import eu.etaxonomy.cdm.model.taxon.Taxon;
  72 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  73 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  74 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  75 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  76 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  77 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  78 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  79 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  80
  81 /**
  82  * @author a.babadshanjan, a.kohlbecker
  83  * @created 04.02.2009
  84  */
  85 public class TaxonServiceSearchTest extends CdmTransactionalIntegrationTest {
  86
  87     private static final String ABIES_BALSAMEA_UUID = "f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8";
  88
  89     private static final String ABIES_ALBA_UUID = "7dbd5810-a3e5-44b6-b563-25152b8867f4";
  90
  91     private static final String CLASSIFICATION_UUID = "2a5ceebb-4830-4524-b330-78461bf8cb6b";
  92
  93     private static final String CLASSIFICATION_ALT_UUID = "d7c741e3-ae9e-4a7d-a566-9e3a7a0b51ce";
  94
  95     private static final String D_ABIES_BALSAMEA_UUID = "900108d8-e6ce-495e-b32e-7aad3099135e";
  96
  97     private static final String D_ABIES_ALBA_UUID = "ec8bba03-d993-4c85-8472-18b14942464b";
  98
  99     private static final String D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID = "e9d8c2fd-6409-46d5-9c2e-14a2bbb1b2b1";
 100
 101     private static final int NUM_OF_NEW_RADOM_ENTITIES = 1000;
 102
 103     private static Logger logger = Logger.getLogger(TaxonServiceSearchTest.class);
 104
 105
 106
 107     @SpringBeanByType
 108     private ITaxonService taxonService;
 109     @SpringBeanByType
 110     private ITermService termService;
 111     @SpringBeanByType
 112     private IClassificationService classificationService;
 113     @SpringBeanByType
 114     private IReferenceService referenceService;
 115     @SpringBeanByType
 116     private IDescriptionService descriptionService;
 117     @SpringBeanByType
 118     private INameService nameService;
 119     @SpringBeanByType
 120     private ICdmMassIndexer indexer;
 121
 122     @SpringBeanByType
 123     private ITaxonNodeService nodeService;
 124
 125     private static final int BENCHMARK_ROUNDS = 300;
 126
 127     private Set<Class<? extends CdmBase>> typesToIndex = null;
 128
 129     private NamedArea germany;
 130     private NamedArea france ;
 131     private NamedArea russia ;
 132     private NamedArea canada ;
 133
 134     /**
 135      * @throws java.lang.Exception
 136      */
 137     @Before
 138     public void setUp() throws Exception {
 139         typesToIndex = new HashSet<Class<? extends CdmBase>>();
 140         typesToIndex.add(DescriptionElementBase.class);
 141         typesToIndex.add(TaxonBase.class);
 142         typesToIndex.add(TaxonRelationship.class);
 143
 144         germany =  Country.GERMANY();
 145         france = Country.FRANCEFRENCHREPUBLIC();
 146         russia = Country.RUSSIANFEDERATION();
 147         canada = Country.CANADA();
 148
 149
 150     }
 151
 152     @Test
 153     public void testDbUnitUsageTest() throws Exception {
 154         assertNotNull("taxonService should exist", taxonService);
 155         assertNotNull("nameService should exist", nameService);
 156     }
 157
 158
 159     @SuppressWarnings("rawtypes")
 160     @Test
 161     @DataSet
 162     public final void testPurgeAndReindex() throws CorruptIndexException, IOException, ParseException {
 163
 164         refreshLuceneIndex();
 165
 166         Pager<SearchResult<TaxonBase>> pager;
 167
 168         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 8
 169         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 170
 171         indexer.purge(null);
 172         commitAndStartNewTransaction(null);
 173
 174         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 0
 175         Assert.assertEquals("Expecting no entities since the index has been purged", 0, pager.getCount().intValue());
 176
 177         indexer.reindex(indexer.indexedClasses(), null);
 178         commitAndStartNewTransaction(null);
 179
 180         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 8
 181         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 182     }
 183
 184
 185     @SuppressWarnings("rawtypes")
 186     @Test
 187     @DataSet
 188     public final void testFindByDescriptionElementFullText_CommonName() throws CorruptIndexException, IOException,
 189             ParseException {
 190
 191         refreshLuceneIndex();
 192
 193         Pager<SearchResult<TaxonBase>> pager;
 194
 195         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null, null,
 196                 false, null, null, null, null);
 197         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1,
 198                 pager.getCount().intValue());
 199
 200         // the description containing the Nulltanne has no taxon attached,
 201         // taxon.id = null
 202         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Nulltanne", null, null, null,
 203                 false, null, null, null, null);
 204         Assert.assertEquals("Expecting no entity when searching for 'Nulltanne' ", 0, pager.getCount().intValue());
 205
 206         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 207                 Arrays.asList(new Language[] { Language.GERMAN() }), false, null, null, null, null);
 208         Assert.assertEquals("Expecting one entity when searching in German", 1, pager.getCount().intValue());
 209
 210         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 211                 Arrays.asList(new Language[] { Language.RUSSIAN() }), false, null, null, null, null);
 212         Assert.assertEquals("Expecting no entity when searching in Russian", 0, pager.getCount().intValue());
 213
 214     }
 215
 216     @SuppressWarnings("rawtypes")
 217     @Test
 218     @DataSet
 219     public final void testFindByDescriptionElementFullText_Distribution() throws CorruptIndexException, IOException, ParseException {
 220
 221         refreshLuceneIndex();
 222
 223         Pager<SearchResult<TaxonBase>> pager;
 224         // by Area
 225         pager = taxonService.findByDescriptionElementFullText(null, "Canada", null, null, null, false, null, null, null, null);
 226         Assert.assertEquals("Expecting one entity when searching for arae 'Canada'", 1, pager.getCount().intValue());
 227         // by Status
 228         pager = taxonService.findByDescriptionElementFullText(null, "present", null, null, null, false, null, null, null, null);
 229         Assert.assertEquals("Expecting one entity when searching for status 'present'", 1, pager.getCount().intValue());
 230     }
 231
 232     @SuppressWarnings("rawtypes")
 233     @Test
 234     @DataSet
 235     public final void testFindByDescriptionElementFullText_wildcard() throws CorruptIndexException, IOException, ParseException {
 236
 237         refreshLuceneIndex();
 238
 239         Pager<SearchResult<TaxonBase>> pager;
 240
 241         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
 242         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", 1, pager.getCount().intValue());
 243     }
 244
 245     /**
 246      * Regression test for #3113 (hibernate search: wildcard query can cause BooleanQuery$TooManyClauses: maxClauseCount is set to 1024)
 247      *
 248      * @throws CorruptIndexException
 249      * @throws IOException
 250      * @throws ParseException
 251      */
 252     @SuppressWarnings("rawtypes")
 253     @Test
 254     @DataSet
 255     public final void testFindByDescriptionElementFullText_TooManyClauses() throws CorruptIndexException, IOException, ParseException {
 256
 257         // generate 1024 terms to reproduce the bug
 258         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 259         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 260         while(uniqueRandomStrs.size() < 1024){
 261             uniqueRandomStrs.add(RandomStringUtils.random(10, true, false));
 262         }
 263         for(String rndStr: uniqueRandomStrs){
 264             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 265         }
 266         descriptionService.saveOrUpdate(description);
 267         commitAndStartNewTransaction(null);
 268
 269         refreshLuceneIndex();
 270
 271         Pager<SearchResult<TaxonBase>> pager;
 272
 273         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, null, null, null, null);
 274         Assert.assertEquals("Expecting all 1024 entities grouped into one SearchResult item when searching for Rot*", 1, pager.getCount().intValue());
 275     }
 276
 277     /**
 278      * Regression test for #3116 (fulltext search: always only one page of results)
 279      *
 280      * @throws CorruptIndexException
 281      * @throws IOException
 282      * @throws ParseException
 283      */
 284     @SuppressWarnings("rawtypes")
 285     @Test
 286     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 287     @Ignore
 288     public final void testFullText_Paging() throws CorruptIndexException, IOException, ParseException {
 289
 290         Reference sec = ReferenceFactory.newDatabase();
 291         referenceService.save(sec);
 292
 293         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 294         int numOfItems = 100;
 295         while(uniqueRandomStrs.size() < numOfItems){
 296             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 297         }
 298
 299         for(String rndStr: uniqueRandomStrs){
 300
 301             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(Rank.SERIES()), sec);
 302             taxon.setTitleCache("Tax" + rndStr, true);
 303             taxonService.save(taxon);
 304
 305             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 306             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 307             descriptionService.saveOrUpdate(description);
 308         }
 309
 310         commitAndStartNewTransaction(new String[]{"TAXONBASE"});
 311         refreshLuceneIndex();
 312
 313         int pageSize = 10;
 314
 315         Pager<SearchResult<TaxonBase>> pager;
 316
 317         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, null, null, null);
 318         Assert.assertEquals("unexpeted number of pages", Integer.valueOf(numOfItems / pageSize), pager.getPagesAvailable());
 319         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 9, null, null);
 320         Assert.assertNotNull("last page must have records", pager.getRecords());
 321         Assert.assertNotNull("last item on last page must exist", pager.getRecords().get(0));
 322         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 10, null, null);
 323         Assert.assertNotNull("last page + 1 must not have any records", pager.getRecords());
 324     }
 325
 326     /**
 327      * test for max score and sort by score of hit groups
 328      * with all matches per taxon in a single TextData  element
 329      * see {@link #testFullText_ScoreAndOrder_2()} for the complement
 330      * test with matches in multiple TextData per taxon
 331      *
 332      * @throws CorruptIndexException
 333      * @throws IOException
 334      * @throws ParseException
 335      */
 336     @SuppressWarnings("rawtypes")
 337     @Test
 338     @DataSet
 339     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 340     public final void testFullText_ScoreAndOrder_1() throws CorruptIndexException, IOException, ParseException {
 341
 342         int numOfTaxa = 3;
 343
 344         UUID[] taxonUuids = new UUID[numOfTaxa];
 345         StringBuilder text = new StringBuilder();
 346
 347         for(int i = 0; i < numOfTaxa; i++){
 348
 349             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(null), null);
 350             taxon.setTitleCache("Taxon_" + i, true);
 351             taxonUuids[i] = taxon.getUuid();
 352             taxonService.save(taxon);
 353
 354             text.append(" ").append("Rot");
 355             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 356             description.addElement(TextData.NewInstance(text.toString(), Language.DEFAULT(), null));
 357             descriptionService.saveOrUpdate(description);
 358         }
 359
 360         commitAndStartNewTransaction(null);
 361         refreshLuceneIndex();
 362
 363         Pager<SearchResult<TaxonBase>> pager;
 364
 365         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 366         for(int i = 0; i < numOfTaxa; i++){
 367             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 368         }
 369         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 370     }
 371
 372     /**
 373      * test for max score and sort by score of hit groups
 374      * with all matches per taxon in a multiple TextData elements
 375      * see {@link #testFullText_ScoreAndOrder_1()} for the complement
 376      * test with matches in a single TextData per taxon
 377      *
 378      * @throws CorruptIndexException
 379      * @throws IOException
 380      * @throws ParseException
 381      */
 382     @SuppressWarnings("rawtypes")
 383     @Test
 384     @DataSet
 385     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 386     public final void testFullText_ScoreAndOrder_2() throws CorruptIndexException, IOException, ParseException {
 387
 388         int numOfTaxa = 3;
 389
 390         UUID[] taxonUuids = new UUID[numOfTaxa];
 391
 392         for(int i = 0; i < numOfTaxa; i++){
 393
 394             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(null), null);
 395             taxon.setTitleCache("Taxon_" + i, true);
 396             taxonUuids[i] = taxon.getUuid();
 397             taxonService.save(taxon);
 398
 399             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 400             for(int k = 0; k < i; k++){
 401                 description.addElement(TextData.NewInstance("Rot", Language.DEFAULT(), null));
 402             }
 403             descriptionService.saveOrUpdate(description);
 404         }
 405
 406         commitAndStartNewTransaction(null);
 407         refreshLuceneIndex();
 408
 409         Pager<SearchResult<TaxonBase>> pager;
 410
 411         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 412         for(int i = 0; i < numOfTaxa; i++){
 413             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 414         }
 415         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 416     }
 417
 418
 419     /**
 420      * @throws CorruptIndexException
 421      * @throws IOException
 422      * @throws ParseException
 423      * @throws LuceneMultiSearchException
 424      */
 425     @Test
 426     @DataSet
 427     public final void testFullText_Grouping() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 428
 429         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 430         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 431         int numOfItems = 100;
 432         while(uniqueRandomStrs.size() < numOfItems){
 433             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 434         }
 435         for(String rndStr: uniqueRandomStrs){
 436             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 437         }
 438         descriptionService.saveOrUpdate(description);
 439
 440         commitAndStartNewTransaction(new String[]{"DESCRIPTIONELEMENTBASE"});
 441
 442         refreshLuceneIndex();
 443
 444         int pageSize = 10;
 445
 446         Pager<SearchResult<TaxonBase>> pager;
 447         boolean highlightFragments = true;
 448
 449         // test with findByDescriptionElementFullText
 450         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, highlightFragments, pageSize, null, null, null);
 451         logFreeTextSearchResults(pager, Level.DEBUG, null);
 452         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 453         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 454         Map<String, String[]> highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 455         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 456         int maxDocsPerGroup = 10;
 457         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 458
 459         // test with findByEverythingFullText
 460         pager = taxonService.findByEverythingFullText( "Rot*", null, null, highlightFragments, pageSize, null, null, null);
 461         logFreeTextSearchResults(pager, Level.DEBUG, null);
 462         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 463         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 464         highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 465         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 466         maxDocsPerGroup = 10;
 467         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 468
 469     }
 470
 471     @SuppressWarnings("rawtypes")
 472     @Test
 473     @DataSet
 474     @Ignore
 475     public final void testFindByDescriptionElementFullText_TextData() throws CorruptIndexException, IOException, ParseException {
 476
 477         refreshLuceneIndex();
 478
 479         Pager<SearchResult<TaxonBase>> pager;
 480         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, false, null, null, null, null);
 481         logFreeTextSearchResults(pager, Level.DEBUG, null);
 482         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 483         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 484         Assert.assertTrue("Expecting two docs, one for RUSSIAN and one for GERMAN", pager.getRecords().get(0).getDocs().size() == 2);
 485         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 486
 487
 488         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, null, null, false, null, null, null, null);
 489         Assert.assertEquals("Expecting one entity when searching for any type", 1, pager.getCount().intValue());
 490
 491         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.UNKNOWN()}), null, false, null, null, null, null);
 492         Assert.assertEquals("Expecting one entity when searching for any type and for Feature DESCRIPTION", 1, pager.getCount().intValue());
 493
 494         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER()}), null, false, null, null, null, null);
 495         Assert.assertEquals("Expecting no entity when searching for any type and for Feature CHROMOSOME_NUMBER", 0, pager.getCount().intValue());
 496
 497         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER(), Feature.UNKNOWN()}), null, false, null, null, null, null);
 498         Assert.assertEquals("Expecting no entity when searching for any type and for Feature DESCRIPTION or CHROMOSOME_NUMBER", 1, pager.getCount().intValue());
 499
 500         pager = taxonService.findByDescriptionElementFullText(Distribution.class, "Abies", null, null, null, false, null, null, null, null);
 501         Assert.assertEquals("Expecting no entity when searching for Distribution", 0, pager.getCount().intValue());
 502
 503         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{}), false, null, null, null, null);
 504         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 505         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 506
 507         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.RUSSIAN()}), false, null, null, null, null);
 508         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 509         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 510
 511         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 512         Assert.assertEquals("Expecting no entity", 0, pager.getCount().intValue());
 513
 514         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 515         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 516         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 517     }
 518
 519     @SuppressWarnings("rawtypes")
 520     @Test
 521     @DataSet
 522     public final void testFindByDescriptionElementFullText_MultipleWords() throws CorruptIndexException, IOException, ParseException {
 523
 524         refreshLuceneIndex();
 525
 526         // Pflanzenart aus der Gattung der Tannen
 527         long start = System.currentTimeMillis();
 528
 529         Pager<SearchResult<TaxonBase>> pager;
 530         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, false, null, null, null, null);
 531         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 532
 533         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Wespen", null, null, null, false, null, null, null, null);
 534         Assert.assertEquals("OR search : Expecting one entity", 1, pager.getCount().intValue());
 535
 536         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, false, null, null, null, null);
 537         Assert.assertEquals("AND search : Expecting one entity", 1, pager.getCount().intValue());
 538
 539         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Wespen", null, null, null, false, null, null, null, null);
 540         Assert.assertEquals("AND search : Expecting no entity", 0, pager.getCount().intValue());
 541
 542         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, false, null, null, null, null);
 543         Assert.assertEquals("Phrase search : Expecting one entity", 1, pager.getCount().intValue());
 544
 545         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Wespen\"", null, null, null, false, null, null, null, null);
 546         Assert.assertEquals("Phrase search : Expecting one entity", 0, pager.getCount().intValue());
 547
 548         logger.info("testFindByDescriptionElementFullText_MultipleWords() duration: " + (System.currentTimeMillis() - start) + "ms");
 549
 550     }
 551
 552
 553     @SuppressWarnings("rawtypes")
 554     @Test
 555     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 556     public final void testFindByDescriptionElementFullText_modify_DescriptionElement() throws CorruptIndexException, IOException, ParseException {
 557
 558         refreshLuceneIndex();
 559
 560         Pager<SearchResult<TaxonBase>> pager;
 561         //
 562         // modify the DescriptionElement
 563         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 564         Assert.assertTrue("Search did not return any results", pager.getRecords().size() > 0);
 565         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 566         Document indexDocument = pager.getRecords().get(0).getDocs().iterator().next();
 567         String[] descriptionElementUuidStr = indexDocument.getValues("uuid");
 568         String[] inDescriptionUuidStr = indexDocument.getValues("inDescription.uuid");
 569         // is only one uuid!
 570         DescriptionElementBase textData = descriptionService.getDescriptionElementByUuid(UUID.fromString(descriptionElementUuidStr[0]));
 571
 572         ((TextData)textData).removeText(Language.GERMAN());
 573         ((TextData)textData).putText(Language.SPANISH_CASTILIAN(), "abeto bals"+UTF8.SMALL_A_ACUTE+"mico");
 574
 575         descriptionService.saveDescriptionElement(textData);
 576         commitAndStartNewTransaction(null);
 577 //        printDataSet(System.out, new String[] {
 578 //                "DESCRIPTIONELEMENTBASE", "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING" }
 579 //        );
 580
 581         //
 582         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 583         Assert.assertEquals("The german 'Balsam-Tanne' TextData should no longer be indexed", 0, pager.getCount().intValue());
 584         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 585         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 586         pager = taxonService.findByDescriptionElementFullText(TextData.class, "bals"+UTF8.SMALL_A_ACUTE+"mico", null, null, null, false, null, null, null, null);
 587         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico'", 1, pager.getCount().intValue());
 588
 589         //
 590         // modify the DescriptionElement via the Description object
 591         DescriptionBase<?> description = descriptionService.find(UUID.fromString(inDescriptionUuidStr[0]));
 592         Set<DescriptionElementBase> elements = description.getElements();
 593         for( DescriptionElementBase elm : elements){
 594             if(elm.getUuid().toString().equals(descriptionElementUuidStr[0])){
 595                 ((TextData)elm).removeText(Language.SPANISH_CASTILIAN());
 596                 ((TextData)elm).putText(Language.POLISH(), "Jod"+UTF8.POLISH_L+"a balsamiczna");
 597             }
 598         }
 599         descriptionService.saveOrUpdate(description);
 600         commitAndStartNewTransaction(null);
 601         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 602         Assert.assertEquals("The spanish 'abeto bals"+UTF8.SMALL_A_ACUTE+"mico' TextData should no longer be indexed", 0, pager.getCount().intValue());
 603         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsamiczna", null, null, Arrays.asList(new Language[]{Language.POLISH()}), false, null, null, null, null);
 604         Assert.assertEquals("expecting to find the POLISH 'Jod"+UTF8.POLISH_L+"a balsamiczna'", 1, pager.getCount().intValue());
 605     }
 606
 607     @SuppressWarnings("rawtypes")
 608     @Test
 609     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 610     public final void testFindByDescriptionElementFullText_modify_Taxon() throws CorruptIndexException, IOException, ParseException {
 611
 612         refreshLuceneIndex();
 613
 614         Pager<SearchResult<TaxonBase>> pager;
 615         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
 616         TaxonDescription d_abies_balsamea = (TaxonDescription)descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 617
 618         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 619         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne'", 1, pager.getCount().intValue());
 620
 621         // exchange the Taxon with another one via the Taxon object
 622         // 1.) remove existing description:
 623         t_abies_balsamea.removeDescription(d_abies_balsamea);
 624
 625         taxonService.saveOrUpdate(t_abies_balsamea);
 626         commitAndStartNewTransaction(null);
 627
 628         t_abies_balsamea = (Taxon)taxonService.find(t_abies_balsamea.getUuid());
 629
 630         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 631         Assert.assertEquals("'Balsam-Tanne' should no longer be found", 0, pager.getCount().intValue());
 632
 633         // 2.) create new description and add to taxon:
 634         TaxonDescription d_abies_balsamea_new = TaxonDescription.NewInstance();
 635         d_abies_balsamea_new
 636                 .addElement(TextData
 637                         .NewInstance(
 638                                 "Die Balsamtanne ist mit bis zu 30 m Höhe ein mittelgro"+UTF8.SHARP_S+"er Baum und kann bis zu 200 Jahre alt werden",
 639                                 Language.GERMAN(), null));
 640         t_abies_balsamea.addDescription(d_abies_balsamea_new);
 641         // set authorshipCache to null to avoid validation exception,
 642         // this is maybe not needed in future,  see ticket #3344
 643         BotanicalName abies_balsamea = HibernateProxyHelper.deproxy(t_abies_balsamea.getName(), BotanicalName.class);
 644         abies_balsamea.setAuthorshipCache(null);
 645         printDataSet(System.err, new String[] {"LANGUAGESTRING_AUD"});
 646         taxonService.saveOrUpdate(t_abies_balsamea);
 647         commitAndStartNewTransaction(null);
 648
 649 //        printDataSet(System.out, new String[] {
 650 //                "DESCRIPTIONBASE"
 651 //        });
 652
 653         pager = taxonService.findByDescriptionElementFullText(TextData.class, "mittelgro"+UTF8.SHARP_S+"er Baum", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 654         Assert.assertEquals("the taxon should be found via the new Description", 1, pager.getCount().intValue());
 655     }
 656
 657     @SuppressWarnings("rawtypes")
 658     @Test
 659     @DataSet
 660     public final void testFindByDescriptionElementFullText_modify_Classification() throws CorruptIndexException, IOException, ParseException {
 661
 662         refreshLuceneIndex();
 663
 664         Pager<SearchResult<TaxonBase>> pager;
 665
 666         // put taxon into other classification, new taxon node
 667         Classification classification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 668         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 669
 670         // TODO: why is the test failing when the childNode is already retrieved here, and not after the following four lines?
 671         //TaxonNode childNode = classification.getChildNodes().iterator().next();
 672
 673         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 674         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne' even if filtering by classification", 1, pager.getCount().intValue());
 675         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 676         Assert.assertEquals("GERMAN 'Balsam-Tanne' should NOT be found in other classification", 0, pager.getCount().intValue());
 677
 678         // check for the right taxon node
 679         TaxonNode childNode = classification.getChildNodes().iterator().next();
 680         Assert.assertEquals("expecting Abies balsamea sec.", childNode.getTaxon().getUuid().toString(), ABIES_BALSAMEA_UUID);
 681         Assert.assertEquals("expecting default classification", childNode.getClassification().getUuid().toString(), CLASSIFICATION_UUID);
 682
 683         // moving the taxon around, the rootnode is only a proxy
 684         alternateClassification.setRootNode(HibernateProxyHelper.deproxy(alternateClassification.getRootNode(), TaxonNode.class));
 685         alternateClassification.addChildNode(childNode, null, null);
 686
 687         classificationService.saveOrUpdate(alternateClassification);
 688         commitAndStartNewTransaction(null);
 689
 690 //        printDataSet(System.out, new String[] {
 691 //            "TAXONBASE", "TAXONNODE", "CLASSIFICATION"
 692 //        });
 693
 694         // reload classification
 695         classification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 696         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 697         Assert.assertEquals("GERMAN 'Balsam-Tanne' should now be found in other classification", 1, pager.getCount().intValue());
 698
 699         classification.getChildNodes().clear();
 700         classificationService.saveOrUpdate(classification);
 701         commitAndStartNewTransaction(null);
 702
 703         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", classification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 704         Assert.assertEquals("Now the GERMAN 'Balsam-Tanne' should NOT be found in original classification", 0, pager.getCount().intValue());
 705
 706     }
 707
 708     @SuppressWarnings("rawtypes")
 709     @Test
 710     @DataSet
 711     public final void testFindByDescriptionElementFullText_CategoricalData() throws CorruptIndexException, IOException, ParseException {
 712
 713         // add CategoricalData
 714         DescriptionBase d_abies_balsamea = descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 715         // Categorical data
 716         CategoricalData cdata = CategoricalData.NewInstance();
 717         cdata.setFeature(Feature.DESCRIPTION());
 718         State state = State.NewInstance("green", "green", "gn");
 719
 720         StateData statedata = StateData.NewInstance(state);
 721         statedata.putModifyingText(Language.ENGLISH(), "always, even during winter");
 722         cdata.addStateData(statedata);
 723         d_abies_balsamea.addElement(cdata);
 724
 725         UUID termUUID = termService.save(state).getUuid();
 726         descriptionService.save(d_abies_balsamea);
 727
 728         commitAndStartNewTransaction(null);
 729
 730 //        printDataSet(System.out, new String[] {
 731 //                 "STATEDATA", "STATEDATA_DEFINEDTERMBASE", "STATEDATA_LANGUAGESTRING", "LANGUAGESTRING"});
 732
 733         refreshLuceneIndex();
 734
 735         Pager<SearchResult<TaxonBase>> pager;
 736         pager = taxonService.findByDescriptionElementFullText(CategoricalData.class, "green", null, null, null, false, null, null, null, null);
 737         Assert.assertEquals("Expecting one entity", 1, pager.getCount().intValue());
 738         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 739         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 740         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 741
 742
 743         //TODO modify the StateData
 744         TaxonBase taxon = pager.getRecords().get(0).getEntity();
 745
 746         String newName = "Quercus robur";
 747         taxon.setTitleCache(newName + " sec. ", true);
 748
 749         taxonService.saveOrUpdate(taxon);
 750         commitAndStartNewTransaction(null);
 751
 752         taxon = taxonService.find(taxon.getUuid());
 753         Assert.assertEquals(newName + " sec. ", taxon.getTitleCache());
 754         DefinedTermBase term = termService.find(termUUID);
 755
 756         termService.delete(term);
 757
 758     }
 759
 760     @SuppressWarnings("rawtypes")
 761     @Test
 762     @DataSet
 763     public final void testFindByDescriptionElementFullText_Highlighting() throws CorruptIndexException, IOException, ParseException {
 764
 765         refreshLuceneIndex();
 766
 767         Pager<SearchResult<TaxonBase>> pager;
 768         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, true, null, null, null, null);
 769         Assert.assertEquals("Expecting one entity when searching for any TextData", 1, pager.getCount().intValue());
 770         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 771         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
 772         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 773         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Abies</B>"));
 774
 775         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, true, null, null, null, null);
 776         searchResult = pager.getRecords().get(0);
 777         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 778         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 779         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") || fragments[0].contains("<B>Tannen</B>"));
 780
 781         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, true, null, null, null, null);
 782         searchResult = pager.getRecords().get(0);
 783         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 784         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 785         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") && fragments[0].contains("<B>Tannen</B>"));
 786
 787         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, true, null, null, null, null);
 788         searchResult = pager.getRecords().get(0);
 789         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 790         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 791         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B> <B>aus</B> <B>der</B> <B>Gattung</B> <B>der</B> <B>Tannen</B>"));
 792
 793         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Gatt*", null, null, null, true, null, null, null, null);
 794         searchResult = pager.getRecords().get(0);
 795         Assert.assertTrue("Wildcard search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 796         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 797         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Gatt"));
 798     }
 799
 800
 801     @Test
 802     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class)
 803     public final void testFindByFullText() throws CorruptIndexException, IOException, ParseException {
 804
 805         refreshLuceneIndex();
 806
 807         Classification europeanAbiesClassification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 808
 809         Pager<SearchResult<TaxonBase>> pager;
 810
 811         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 7
 812         logFreeTextSearchResults(pager, Level.DEBUG, null);
 813         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
 814
 815         pager = taxonService.findByFullText(Taxon.class, "Abies", null, null, true, null, null, null, null); // --> 6
 816         Assert.assertEquals("Expecting 7 entities", 7, pager.getCount().intValue());
 817
 818         pager = taxonService.findByFullText(Synonym.class, "Abies", null, null, true, null, null, null, null); // --> 1
 819         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 820
 821         pager = taxonService.findByFullText(TaxonBase.class, "sec", null, null, true, null, null, null, null); // --> 7
 822         Assert.assertEquals("Expecting 8 entities", 9, pager.getCount().intValue());
 823
 824         pager = taxonService.findByFullText(null, "genus", null, null, true, null, null, null, null); // --> 1
 825         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 826
 827         pager = taxonService.findByFullText(Taxon.class, "subalpina", null, null, true, null, null, null, null); // --> 0
 828         Assert.assertEquals("Expecting 0 entities", 0, pager.getCount().intValue());
 829
 830         // synonym in classification ???
 831     }
 832
 833     @Test
 834     @DataSet
 835     public final void testPrepareByAreaSearch() throws IOException, ParseException {
 836
 837         List<PresenceAbsenceTerm> statusFilter = new ArrayList<PresenceAbsenceTerm>();
 838         List<NamedArea> areaFilter = new ArrayList<NamedArea>();
 839         areaFilter.add(germany);
 840         areaFilter.add(canada);
 841         areaFilter.add(russia);
 842
 843         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDistribution(areaFilter, statusFilter, null, 20, 0, null, null);
 844         Assert.assertEquals("Expecting 2 entities", Integer.valueOf(2), Integer.valueOf(pager.getRecords().size()));
 845
 846     }
 847
 848     @Test
 849     @DataSet
 850     public final void testFindTaxaAndNamesByFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 851
 852         refreshLuceneIndex();
 853
 854         Pager<SearchResult<TaxonBase>> pager;
 855
 856         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 857
 858
 859         pager = taxonService.findTaxaAndNamesByFullText(
 860                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 861                 "Abies", null, null, null, null, true, null, null, null, null);
 862 //        logPagerRecords(pager, Level.DEBUG);
 863         Assert.assertEquals("doTaxa & doSynonyms", 8, pager.getCount().intValue());
 864
 865         pager = taxonService.findTaxaAndNamesByFullText(
 866                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 867                 "Abies", null, null, null, null, true, null, null, null, null);
 868 //        logPagerRecords(pager, Level.DEBUG);
 869         Assert.assertEquals("all search modes", 8, pager.getCount().intValue());
 870
 871         pager = taxonService.findTaxaAndNamesByFullText(
 872                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 873                 "Abies", alternateClassification, null, null, null, true, null, null, null, null);
 874 //        logPagerRecords(pager, Level.DEBUG);
 875         Assert.assertEquals("all search modes, filtered by alternateClassification", 1, pager.getCount().intValue());
 876
 877         pager = taxonService.findTaxaAndNamesByFullText(
 878                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
 879                 "Abies", null, null, null, null, true, null, null, null, null);
 880         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 881         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 882         Assert.assertEquals(Synonym.class, searchResult.getEntity().getClass());
 883         // Abies subalpina sec. Kohlbecker, A., Testcase standart views, 2013
 884
 885
 886         pager = taxonService.findTaxaAndNamesByFullText(
 887                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 888                 "Abies", null, null, null, null, true, null, null, null, null);
 889         Assert.assertEquals("Expecting 0 entity", 0, pager.getCount().intValue());
 890
 891
 892         pager = taxonService.findTaxaAndNamesByFullText(
 893                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 894                 "Tanne", null, null, null, null, true, null, null, null, null);
 895         Assert.assertEquals("Expecting 1 entity", 1, pager.getRecords().size());
 896         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 897
 898         pager = taxonService.findTaxaAndNamesByFullText(
 899                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
 900                 "kawakamii", null, null, null, null, true, null, null, null, null);
 901         logFreeTextSearchResults(pager, Level.DEBUG, null);
 902         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
 903
 904     }
 905
 906     @Test
 907     @DataSet
 908     public final void testFindTaxaAndNamesByFullText_PhraseQuery() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 909
 910         refreshLuceneIndex();
 911
 912         Pager<SearchResult<TaxonBase>> pager;
 913
 914
 915         pager = taxonService.findTaxaAndNamesByFullText(
 916                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 917                 "\"Abies alba\"", null, null, null, null, true, null, null, null, null);
 918 //        logPagerRecords(pager, Level.DEBUG);
 919         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 1, pager.getCount().intValue());
 920
 921         pager = taxonService.findTaxaAndNamesByFullText(
 922                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 923                 "\"Abies al*\"", null, null, null, null, true, null, null, null, null);
 924 //        logPagerRecords(pager, Level.DEBUG);
 925         Assert.assertEquals("doTaxa & doSynonyms with complex phrase query", 1, pager.getCount().intValue());
 926
 927         pager = taxonService.findTaxaAndNamesByFullText(
 928                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 929                 "\"Abies*\"", null, null, null, null, true, null, null, null, null);
 930 //        logPagerRecords(pager, Level.DEBUG);
 931         Assert.assertEquals("doTaxa & doSynonyms with simple phrase query", 8, pager.getCount().intValue());
 932
 933     }
 934
 935     @Test
 936     @DataSet
 937     public final void testFindTaxaAndNamesByFullText_Sort() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 938
 939         refreshLuceneIndex();
 940
 941         Pager<SearchResult<TaxonBase>> pager;
 942
 943         List<OrderHint> orderHints = new ArrayList<OrderHint>();
 944
 945         String[] docFields2log = new String[]{"id"};
 946
 947         // SortById
 948         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
 949         pager = taxonService.findTaxaAndNamesByFullText(
 950                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
 951                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 952 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 953         int lastId = -1;
 954         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 955             if(lastId != -1){
 956                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
 957             }
 958             lastId = rs.getEntity().getId();
 959         }
 960
 961         orderHints.addAll(OrderHint.ORDER_BY_ID.asList());
 962         pager = taxonService.findTaxaAndNamesByFullText(
 963                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 964                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 965 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 966
 967         lastId = -1;
 968         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 969             if(lastId != -1){
 970                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
 971             }
 972             lastId = rs.getEntity().getId();
 973         }
 974
 975         // Sortby NOMENCLATURAL_SORT_ORDER TODO make assertions !!!
 976         orderHints.clear();
 977         orderHints.addAll(OrderHint.NOMENCLATURAL_SORT_ORDER.asList());
 978         pager = taxonService.findTaxaAndNamesByFullText(
 979                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 980                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 981         logFreeTextSearchResults(pager, Level.DEBUG, null);
 982
 983     }
 984
 985     @Test
 986     @DataSet
 987     public final void testFindTaxaAndNamesByFullText_AreaFilter() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 988
 989         refreshLuceneIndex();
 990
 991         Pager<SearchResult<TaxonBase>> pager;
 992
 993         Set<NamedArea> a_germany_canada_russia = new HashSet<NamedArea>();
 994         a_germany_canada_russia.add(germany);
 995         a_germany_canada_russia.add(canada);
 996         a_germany_canada_russia.add(russia);
 997
 998         Set<NamedArea> a_russia = new HashSet<NamedArea>();
 999         a_russia.add(russia);
1000
1001         Set<PresenceAbsenceTerm> present = new HashSet<PresenceAbsenceTerm>();
1002         present.add(PresenceAbsenceTerm.PRESENT());
1003
1004         Set<PresenceAbsenceTerm> present_native = new HashSet<PresenceAbsenceTerm>();
1005         present_native.add(PresenceAbsenceTerm.PRESENT());
1006         present_native.add(PresenceAbsenceTerm.NATIVE());
1007
1008         Set<PresenceAbsenceTerm> absent = new HashSet<PresenceAbsenceTerm>();
1009         absent.add(PresenceAbsenceTerm.ABSENT());
1010
1011         pager = taxonService.findTaxaAndNamesByFullText(
1012                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
1013                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1014         logFreeTextSearchResults(pager, Level.DEBUG, null);
1015
1016         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1017         pager = taxonService.findTaxaAndNamesByFullText(
1018                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
1019                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1020         Assert.assertEquals("synonyms with matching area filter", 1, pager.getCount().intValue());
1021
1022         pager = taxonService.findTaxaAndNamesByFullText(
1023                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1024                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1025         logFreeTextSearchResults(pager, Level.DEBUG, null);
1026         Assert.assertEquals("taxa and synonyms with matching area filter", 3, pager.getCount().intValue());
1027
1028         pager = taxonService.findTaxaAndNamesByFullText(
1029                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1030                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1031         Assert.assertEquals("taxa and synonyms with matching area & status filter 1", 3, pager.getCount().intValue());
1032
1033         pager = taxonService.findTaxaAndNamesByFullText(
1034                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1035                 "Abies", null, a_germany_canada_russia, present, null, true, null, null, null, null);
1036         Assert.assertEquals("taxa and synonyms with matching area & status filter 2", 2, pager.getCount().intValue());
1037
1038         pager = taxonService.findTaxaAndNamesByFullText(
1039                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1040                 "Abies", null, a_russia, present, null, true, null, null, null, null);
1041         Assert.assertEquals("taxa and synonyms with non matching area & status filter", 0, pager.getCount().intValue());
1042
1043         pager = taxonService.findTaxaAndNamesByFullText(
1044                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
1045                 "Tanne", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1046         Assert.assertEquals("ByCommonNames with area filter", 1, pager.getCount().intValue());
1047
1048         // abies_kawakamii_sensu_komarov as misapplied name for t_abies_balsamea
1049         pager = taxonService.findTaxaAndNamesByFullText(
1050                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1051                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1052         Assert.assertEquals("misappliedNames with matching area & status filter", 1, pager.getCount().intValue());
1053
1054
1055         // 1. remove existing taxon relation
1056         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
1057         Set<TaxonRelationship> relsTo = t_abies_balsamea.getRelationsToThisTaxon();
1058         Assert.assertEquals(1, relsTo.size());
1059         TaxonRelationship taxonRelation = relsTo.iterator().next();
1060         t_abies_balsamea.removeTaxonRelation(taxonRelation);
1061         taxonService.saveOrUpdate(t_abies_balsamea);
1062         commitAndStartNewTransaction(null);
1063
1064         pager = taxonService.findTaxaAndNamesByFullText(
1065                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1066                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1067         Assert.assertEquals("misappliedNames with matching area & status filter, should match nothing now", 0, pager.getCount().intValue());
1068
1069         // 2. now add abies_kawakamii_sensu_komarov as misapplied name for t_abies_alba and search for misapplications in russia: ABSENT
1070         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(UUID.fromString(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1071         Taxon t_abies_alba = (Taxon)taxonService.find(UUID.fromString(ABIES_ALBA_UUID));
1072         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1073
1074         taxonService.update(t_abies_kawakamii_sensu_komarov);
1075
1076         commitAndStartNewTransaction(null);
1077
1078         pager = taxonService.findTaxaAndNamesByFullText(
1079                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1080                 "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1081         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1082
1083     }
1084
1085     @Test
1086     @DataSet
1087     @Ignore // remove once http://dev.e-taxonomy.eu/trac/ticket/5477 is solved
1088     public final void testFindTaxaAndNamesByFullText_AreaFilter_issue5477() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1089
1090         Set<NamedArea> a_germany_canada_russia = new HashSet<NamedArea>();
1091         a_germany_canada_russia.add(germany);
1092         a_germany_canada_russia.add(canada);
1093         a_germany_canada_russia.add(russia);
1094
1095
1096         Set<PresenceAbsenceTerm> absent = new HashSet<PresenceAbsenceTerm>();
1097         absent.add(PresenceAbsenceTerm.ABSENT());
1098
1099         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(UUID.fromString(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1100         Taxon t_abies_alba = (Taxon)taxonService.find(UUID.fromString(ABIES_ALBA_UUID));
1101         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1102
1103         /* Since the upgrade from hibernate search 4 to 5.5
1104          * triggering an update of t_abies_alba is no longer sufficient to also update the
1105          * document of t_abies_kawakamii_sensu_komarov in the lucene index.
1106          * the last test in testFindTaxaAndNamesByFullText_AreaFilter() failed in this case.
1107          * This situation is reproduced here:
1108          */
1109         taxonService.update(t_abies_alba);
1110
1111           commitAndStartNewTransaction(null);
1112
1113           Pager pager = taxonService.findTaxaAndNamesByFullText(
1114                   EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1115                   "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1116           Assert.assertEquals("misappliedNames with matching area & status filter, should find one", 1, pager.getCount().intValue());
1117     }
1118
1119
1120     /**
1121      * Regression test for #3119: fulltext search: Entity always null whatever search
1122      *
1123      * @throws CorruptIndexException
1124      * @throws IOException
1125      * @throws ParseException
1126      * @throws LuceneMultiSearchException
1127      */
1128     @Test
1129     @DataSet
1130     public final void testFindByEverythingFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1131
1132         refreshLuceneIndex();
1133
1134         Pager<SearchResult<TaxonBase>> pager;
1135
1136         // via Taxon
1137         pager = taxonService.findByEverythingFullText("Abies", null, null, true, null, null, null, null);
1138         logFreeTextSearchResults(pager, Level.DEBUG, null);
1139         Assert.assertTrue("Expecting at least 7 entities for 'Abies'", pager.getCount() > 7);
1140         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1141         Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1142
1143         // via DescriptionElement
1144         pager = taxonService.findByEverythingFullText("present", null, null, true, null, null, null, null);
1145         Assert.assertEquals("Expecting one entity when searching for area 'present'", 1, pager.getCount().intValue());
1146         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1147         Assert.assertEquals("Expecting Taxon entity", Taxon.class, CdmBase.deproxy(pager.getRecords().get(0).getEntity()).getClass());
1148         Assert.assertEquals("Expecting Taxon ", ABIES_BALSAMEA_UUID, pager.getRecords().get(0).getEntity().getUuid().toString());
1149
1150     }
1151
1152
1153     @Test
1154     @DataSet
1155     public final void findByEveryThingFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1156
1157         refreshLuceneIndex();
1158
1159         Pager<SearchResult<TaxonBase>> pager;
1160
1161         pager = taxonService.findByEverythingFullText("genus", null, null,  false, null, null, null, null); // --> 1
1162         Assert.assertEquals("Expecting 1 entity", 1, pager.getCount().intValue());
1163
1164         //FIXME FAILS: abies balamea is returned twice, see also testFullText_Grouping()
1165         pager = taxonService.findByEverythingFullText("Balsam", null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1166         logFreeTextSearchResults(pager, Level.DEBUG, null);
1167         Assert.assertEquals("expecting to find the Abies balsamea via the GERMAN DescriptionElements", 1, pager.getCount().intValue());
1168
1169         pager = taxonService.findByEverythingFullText("Abies", null, null, true, null, null, null, null);
1170         Assert.assertEquals("Expecting 8 entities", 8, pager.getCount().intValue());
1171         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
1172         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
1173         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
1174         Assert.assertTrue("first fragments should contains serch term", fragments[0].toLowerCase().contains("<b>abies</b>"));
1175     }
1176
1177 //    @SuppressWarnings("rawtypes")
1178 //    @Test
1179 //    @DataSet
1180 //    public final void benchmarkFindTaxaAndNamesHql() throws CorruptIndexException, IOException, ParseException {
1181 //
1182 //        createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1183 //
1184 //        IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1185 //        configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1186 //        configurator.setMatchMode(MatchMode.BEGINNING);
1187 //        configurator.setDoTaxa(false);
1188 //        configurator.setDoSynonyms(false);
1189 //        configurator.setDoNamesWithoutTaxa(false);
1190 //        configurator.setDoTaxaByCommonNames(true);
1191 //
1192 //        Pager<IdentifiableEntity> pager;
1193 //
1194 //        long startMillis = System.currentTimeMillis();
1195 //        for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1196 //            pager = taxonService.findTaxaAndNames(configurator);
1197 //            if (logger.isDebugEnabled()) {
1198 //                logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1199 //            }
1200 //        }
1201 //        double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1202 //        logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1203 //    }
1204
1205     @SuppressWarnings("rawtypes")
1206     @Test
1207     @DataSet
1208     public final void benchmarkFindByCommonNameHql() throws CorruptIndexException, IOException, ParseException {
1209
1210 //        printDataSet(System.err, new String[] { "TaxonBase" });
1211
1212         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1213
1214         IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl<>();
1215         configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1216         configurator.setMatchMode(MatchMode.BEGINNING);
1217         configurator.setDoTaxa(false);
1218         configurator.setDoSynonyms(false);
1219         configurator.setDoNamesWithoutTaxa(false);
1220         configurator.setDoTaxaByCommonNames(true);
1221
1222         Pager<IdentifiableEntity> pager;
1223
1224         long startMillis = System.currentTimeMillis();
1225         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1226             pager = taxonService.findTaxaAndNames(configurator);
1227             if (logger.isDebugEnabled()) {
1228                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1229             }
1230         }
1231         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1232         logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1233     }
1234
1235     @SuppressWarnings("rawtypes")
1236     @Test
1237     @DataSet
1238     public final void benchmarkFindByCommonNameLucene() throws CorruptIndexException, IOException, ParseException {
1239
1240         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1241
1242         refreshLuceneIndex();
1243
1244         Pager<SearchResult<TaxonBase>> pager;
1245
1246         long startMillis = System.currentTimeMillis();
1247         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1248             pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
1249             if (logger.isDebugEnabled()) {
1250                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getEntity().getTitleCache());
1251             }
1252         }
1253         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1254         logger.info("Benchmark result - [find taxon by CommonName via lucene] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1255     }
1256
1257     /**
1258      * uncomment @Test annotation to create the dataset for this test
1259      */
1260     @Override
1261     //    @Test
1262     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="BlankDataSet.xml")
1263     public final void createTestDataSet() throws FileNotFoundException {
1264
1265         Classification europeanAbiesClassification = Classification.NewInstance("European Abies");
1266         europeanAbiesClassification.setUuid(UUID.fromString(CLASSIFICATION_UUID));
1267         classificationService.save(europeanAbiesClassification);
1268
1269         Classification alternativeClassification = Classification.NewInstance("Abies alternative");
1270         alternativeClassification.setUuid(UUID.fromString(CLASSIFICATION_ALT_UUID));
1271         classificationService.save(alternativeClassification);
1272
1273         Reference sec = ReferenceFactory.newBook();
1274         sec.setTitleCache("Kohlbecker, A., Testcase standart views, 2013", true);
1275         Reference sec_sensu = ReferenceFactory.newBook();
1276         sec_sensu.setTitleCache("Komarov, V. L., Flora SSSR 29", true);
1277         referenceService.save(sec);
1278         referenceService.save(sec_sensu);
1279
1280         BotanicalName n_abies = BotanicalName.NewInstance(Rank.GENUS());
1281         n_abies.setNameCache("Abies", true);
1282         Taxon t_abies = Taxon.NewInstance(n_abies, sec);
1283         taxonService.save(t_abies);
1284
1285         BotanicalName n_abies_alba = BotanicalName.NewInstance(Rank.SPECIES());
1286         n_abies_alba.setNameCache("Abies alba", true);
1287         Taxon t_abies_alba = Taxon.NewInstance(n_abies_alba, sec);
1288         t_abies_alba.setUuid(UUID.fromString(ABIES_ALBA_UUID));
1289         taxonService.save(t_abies_alba);
1290
1291         BotanicalName n_abies_subalpina = BotanicalName.NewInstance(Rank.SPECIES());
1292         n_abies_subalpina.setNameCache("Abies subalpina", true);
1293         Synonym s_abies_subalpina = Synonym.NewInstance(n_abies_subalpina, sec);
1294         taxonService.save(s_abies_subalpina);
1295
1296         BotanicalName n_abies_balsamea = BotanicalName.NewInstance(Rank.SPECIES());
1297         n_abies_balsamea.setNameCache("Abies balsamea", true);
1298         Taxon t_abies_balsamea = Taxon.NewInstance(n_abies_balsamea, sec);
1299         t_abies_balsamea.setUuid(UUID.fromString(ABIES_BALSAMEA_UUID));
1300         t_abies_balsamea.addSynonym(s_abies_subalpina, SynonymType.SYNONYM_OF());
1301         taxonService.save(t_abies_balsamea);
1302
1303         BotanicalName n_abies_grandis = BotanicalName.NewInstance(Rank.SPECIES());
1304         n_abies_grandis.setNameCache("Abies grandis", true);
1305         Taxon t_abies_grandis = Taxon.NewInstance(n_abies_grandis, sec);
1306         taxonService.save(t_abies_grandis);
1307
1308         BotanicalName n_abies_kawakamii = BotanicalName.NewInstance(Rank.SPECIES());
1309         n_abies_kawakamii.setNameCache("Abies kawakamii", true);
1310         Taxon t_abies_kawakamii = Taxon.NewInstance(n_abies_kawakamii, sec);
1311         t_abies_kawakamii.getTitleCache();
1312         taxonService.save(t_abies_kawakamii);
1313
1314         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1315         Taxon t_abies_kawakamii_sensu_komarov = Taxon.NewInstance(n_abies_kawakamii, sec_sensu);
1316         taxonService.save(t_abies_kawakamii_sensu_komarov);
1317         t_abies_kawakamii_sensu_komarov.addTaxonRelation(t_abies_balsamea, TaxonRelationshipType.MISAPPLIED_NAME_FOR(), null, null);
1318         taxonService.saveOrUpdate(t_abies_kawakamii_sensu_komarov);
1319
1320         BotanicalName n_abies_lasiocarpa = BotanicalName.NewInstance(Rank.SPECIES());
1321         n_abies_lasiocarpa.setNameCache("Abies lasiocarpa", true);
1322         Taxon t_abies_lasiocarpa = Taxon.NewInstance(n_abies_lasiocarpa, sec);
1323         taxonService.save(t_abies_lasiocarpa);
1324
1325         // add taxa to classifications
1326         europeanAbiesClassification.addChildTaxon(t_abies_balsamea, null, null);
1327         alternativeClassification.addChildTaxon(t_abies_lasiocarpa, null, null);
1328         classificationService.saveOrUpdate(europeanAbiesClassification);
1329         classificationService.saveOrUpdate(alternativeClassification);
1330
1331         //
1332         // Description
1333         //
1334         TaxonDescription d_abies_alba = TaxonDescription.NewInstance(t_abies_alba);
1335         TaxonDescription d_abies_balsamea = TaxonDescription.NewInstance(t_abies_balsamea);
1336
1337         d_abies_alba.setUuid(UUID.fromString(D_ABIES_ALBA_UUID));
1338         d_abies_balsamea.setUuid(UUID.fromString(D_ABIES_BALSAMEA_UUID));
1339
1340
1341         // CommonTaxonName
1342         d_abies_alba.addElement(CommonTaxonName.NewInstance("Wei"+UTF8.SHARP_S+"tanne", Language.GERMAN()));
1343         d_abies_alba.addElement(CommonTaxonName.NewInstance("silver fir", Language.ENGLISH()));
1344         d_abies_alba.addElement(Distribution
1345                 .NewInstance(
1346                         germany,
1347                         PresenceAbsenceTerm.NATIVE()));
1348         d_abies_alba.addElement(Distribution
1349                 .NewInstance(
1350                         russia,
1351                         PresenceAbsenceTerm.ABSENT()));
1352
1353         // TextData
1354         d_abies_balsamea
1355             .addElement(TextData
1356                     .NewInstance(
1357                             "Die Balsam-Tanne (Abies balsamea) ist eine Pflanzenart aus der Gattung der Tannen (Abies). Sie wÃ¤chst im nordÃ¶stlichen Nordamerika, wo sie sowohl Tief- als auch Bergland besiedelt. Sie gilt als relativ anspruchslos gegenÃ¼ber dem Standort und ist frosthart. In vielen Teilen des natÃ¼rlichen Verbreitungsgebietes stellt sie die Klimaxbaumart dar.",
1358                             Language.GERMAN(), null));
1359         d_abies_balsamea
1360         .addElement(CommonTaxonName
1361                 .NewInstance(
1362                         "Balsam-Tanne",
1363                         Language.GERMAN(), null));
1364
1365         d_abies_balsamea
1366         .addElement(Distribution
1367                 .NewInstance(
1368                         canada,
1369                         PresenceAbsenceTerm.PRESENT()));
1370
1371         d_abies_balsamea
1372         .addElement(Distribution
1373                 .NewInstance(
1374                         germany,
1375                         PresenceAbsenceTerm.NATIVE()));
1376
1377         d_abies_balsamea
1378                 .addElement(TextData
1379                         .NewInstance(
1380                                 TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_LONG,
1381                                 Language.RUSSIAN(), null));
1382         d_abies_balsamea
1383         .addElement(CommonTaxonName
1384                 .NewInstance(
1385                         TaxonServiceSearchTestUtf8Constants.RUSSIAN_ABIES_ALBA_SHORT,
1386                         Language.RUSSIAN(), null));
1387         descriptionService.saveOrUpdate(d_abies_balsamea);
1388
1389         setComplete();
1390         endTransaction();
1391
1392
1393         writeDbUnitDataSetFile(new String[] {
1394             "TAXONBASE", "TAXONNAMEBASE",
1395             "TAXONRELATIONSHIP",
1396             "REFERENCE", "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
1397             "AGENTBASE", "HOMOTYPICALGROUP",
1398             "CLASSIFICATION", "TAXONNODE",
1399             "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING",
1400             "HIBERNATE_SEQUENCES" // IMPORTANT!!!
1401             });
1402
1403     }
1404
1405     /**
1406      *
1407      */
1408     private void refreshLuceneIndex() {
1409
1410 //        commitAndStartNewTransaction(null);
1411         commit();
1412         endTransaction();
1413         indexer.purge(DefaultProgressMonitor.NewInstance());
1414         indexer.reindex(typesToIndex, DefaultProgressMonitor.NewInstance());
1415         startNewTransaction();
1416 //        commitAndStartNewTransaction(null);
1417     }
1418
1419     /**
1420      * @param numberOfNew
1421      *
1422      */
1423     private void createRandomTaxonWithCommonName(int numberOfNew) {
1424
1425         logger.debug(String.format("creating %1$s random taxan with CommonName", numberOfNew));
1426
1427         commitAndStartNewTransaction(null);
1428
1429         Reference sec = ReferenceFactory.newBook();
1430         referenceService.save(sec);
1431
1432         for (int i = numberOfNew; i < numberOfNew; i++) {
1433             RandomStringUtils.randomAlphabetic(10);
1434             String radomName = RandomStringUtils.randomAlphabetic(5) + " " + RandomStringUtils.randomAlphabetic(10);
1435             String radomCommonName = RandomStringUtils.randomAlphabetic(10);
1436
1437             BotanicalName name = BotanicalName.NewInstance(Rank.SPECIES());
1438             name.setNameCache(radomName, true);
1439             Taxon taxon = Taxon.NewInstance(name, sec);
1440             taxonService.save(taxon);
1441
1442             TaxonDescription description = TaxonDescription.NewInstance(taxon);
1443             description.addElement(CommonTaxonName.NewInstance(radomCommonName, Language.GERMAN()));
1444             descriptionService.save(description);
1445         }
1446
1447         commitAndStartNewTransaction(null);
1448     }
1449
1450     private <T extends CdmBase> void logFreeTextSearchResults(Pager<SearchResult<T>> pager, Level level, String[] docFields){
1451         if(level == null){
1452             level = Level.DEBUG;
1453         }
1454         if(logger.isEnabledFor(level)){
1455             StringBuilder b = new StringBuilder();
1456             b.append("\n");
1457             int i = 0;
1458             for(SearchResult sr : pager.getRecords()){
1459
1460                 b.append(" ").append(i++).append(" - ");
1461                 b.append("score:").append(sr.getScore()).append(", ");
1462
1463                 if(docFields != null){
1464                     b.append("docs : ");
1465                     for(Document doc : sr.getDocs()) {
1466                         b.append("<");
1467                         for(String f : docFields){
1468                             b.append(f).append(":").append(Arrays.toString(doc.getValues(f)));
1469                         }
1470                         b.append(">");
1471                     }
1472                 }
1473
1474                 CdmBase entity = sr.getEntity();
1475                 if(entity == null){
1476                     b.append("NULL");
1477                 } else {
1478                     b.append(entity.getClass().getSimpleName()).
1479                         append(" [").append(entity.getId()).
1480                         append(" | ").append(entity.getUuid()).append("] : ").
1481                         append(entity.toString());
1482
1483                 }
1484                 b.append("\n");
1485             }
1486             logger.log(level, b);
1487         }
1488     }
1489
1490 }