cdmlib-services/src/test/java/eu/etaxonomy/cdm/api/service/TaxonServiceSearchTest.java

   1 /**
   2  * Copyright (C) 2009 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import static org.junit.Assert.assertEquals;
  13 import static org.junit.Assert.assertNotNull;
  14
  15 import java.io.FileNotFoundException;
  16 import java.io.IOException;
  17 import java.util.ArrayList;
  18 import java.util.Arrays;
  19 import java.util.EnumSet;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Set;
  24 import java.util.UUID;
  25
  26 import org.apache.commons.lang.RandomStringUtils;
  27 import org.apache.log4j.Level;
  28 import org.apache.log4j.Logger;
  29 import org.apache.lucene.document.Document;
  30 import org.apache.lucene.index.CorruptIndexException;
  31 import org.apache.lucene.queryParser.ParseException;
  32 import org.junit.Assert;
  33 import org.junit.Before;
  34 import org.junit.Ignore;
  35 import org.junit.Test;
  36 import org.unitils.dbunit.annotation.DataSet;
  37 import org.unitils.spring.annotation.SpringBeanByType;
  38
  39 import eu.etaxonomy.cdm.api.service.config.FindTaxaAndNamesConfiguratorImpl;
  40 import eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator;
  41 import eu.etaxonomy.cdm.api.service.exception.ReferencedObjectUndeletableException;
  42 import eu.etaxonomy.cdm.api.service.pager.Pager;
  43 import eu.etaxonomy.cdm.api.service.search.ICdmMassIndexer;
  44 import eu.etaxonomy.cdm.api.service.search.LuceneMultiSearchException;
  45 import eu.etaxonomy.cdm.api.service.search.SearchResult;
  46 import eu.etaxonomy.cdm.common.UTF8;
  47 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  48 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  49 import eu.etaxonomy.cdm.model.common.CdmBase;
  50 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  51 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  52 import eu.etaxonomy.cdm.model.common.Language;
  53 import eu.etaxonomy.cdm.model.description.AbsenceTerm;
  54 import eu.etaxonomy.cdm.model.description.CategoricalData;
  55 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  56 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  57 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  58 import eu.etaxonomy.cdm.model.description.Distribution;
  59 import eu.etaxonomy.cdm.model.description.Feature;
  60 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTermBase;
  61 import eu.etaxonomy.cdm.model.description.PresenceTerm;
  62 import eu.etaxonomy.cdm.model.description.State;
  63 import eu.etaxonomy.cdm.model.description.StateData;
  64 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  65 import eu.etaxonomy.cdm.model.description.TextData;
  66 import eu.etaxonomy.cdm.model.location.Country;
  67 import eu.etaxonomy.cdm.model.location.NamedArea;
  68 import eu.etaxonomy.cdm.model.name.BotanicalName;
  69 import eu.etaxonomy.cdm.model.name.NonViralName;
  70 import eu.etaxonomy.cdm.model.name.Rank;
  71 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  72 import eu.etaxonomy.cdm.model.reference.Reference;
  73 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  74 import eu.etaxonomy.cdm.model.taxon.Classification;
  75 import eu.etaxonomy.cdm.model.taxon.Synonym;
  76 import eu.etaxonomy.cdm.model.taxon.SynonymRelationshipType;
  77 import eu.etaxonomy.cdm.model.taxon.Taxon;
  78 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  79 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  80 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  81 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  82 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  83 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  84 import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
  85 import eu.etaxonomy.cdm.test.unitils.CleanSweepInsertLoadStrategy;
  86
  87
  88 /**
  89  * @author a.babadshanjan, a.kohlbecker
  90  * @created 04.02.2009
  91  */
  92 public class TaxonServiceSearchTest extends CdmTransactionalIntegrationTest {
  93
  94     private static final String ABIES_BALSAMEA_UUID = "f65d47bd-4f49-4ab1-bc4a-bc4551eaa1a8";
  95
  96     private static final String ABIES_ALBA_UUID = "7dbd5810-a3e5-44b6-b563-25152b8867f4";
  97
  98     private static final String CLASSIFICATION_UUID = "2a5ceebb-4830-4524-b330-78461bf8cb6b";
  99
 100     private static final String CLASSIFICATION_ALT_UUID = "d7c741e3-ae9e-4a7d-a566-9e3a7a0b51ce";
 101
 102     private static final String D_ABIES_BALSAMEA_UUID = "900108d8-e6ce-495e-b32e-7aad3099135e";
 103
 104     private static final String D_ABIES_ALBA_UUID = "ec8bba03-d993-4c85-8472-18b14942464b";
 105
 106     private static final String D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID = "e9d8c2fd-6409-46d5-9c2e-14a2bbb1b2b1";
 107     private static final int NUM_OF_NEW_RADOM_ENTITIES = 1000;
 108
 109     private static Logger logger = Logger.getLogger(TaxonServiceSearchTest.class);
 110
 111
 112
 113     @SpringBeanByType
 114     private ITaxonService taxonService;
 115     @SpringBeanByType
 116     private ITermService termService;
 117     @SpringBeanByType
 118     private IClassificationService classificationService;
 119     @SpringBeanByType
 120     private IReferenceService referenceService;
 121     @SpringBeanByType
 122     private IDescriptionService descriptionService;
 123     @SpringBeanByType
 124     private INameService nameService;
 125     @SpringBeanByType
 126     private ICdmMassIndexer indexer;
 127
 128     private static final int BENCHMARK_ROUNDS = 300;
 129
 130     private Set<Class<? extends CdmBase>> typesToIndex = null;
 131
 132     private NamedArea germany;
 133     private NamedArea france ;
 134     private NamedArea russia ;
 135     private NamedArea canada ;
 136
 137     /**
 138      * @throws java.lang.Exception
 139      */
 140     @Before
 141     public void setUp() throws Exception {
 142         typesToIndex = new HashSet<Class<? extends CdmBase>>();
 143         typesToIndex.add(DescriptionElementBase.class);
 144         typesToIndex.add(TaxonBase.class);
 145         typesToIndex.add(TaxonRelationship.class);
 146
 147         germany =  Country.GERMANY();
 148         france = Country.FRANCEFRENCHREPUBLIC();
 149         russia = Country.RUSSIANFEDERATION();
 150         canada = Country.CANADA();
 151
 152     }
 153
 154     @Test
 155     public void testDbUnitUsageTest() throws Exception {
 156         assertNotNull("taxonService should exist", taxonService);
 157         assertNotNull("nameService should exist", nameService);
 158     }
 159
 160     /**
 161      * Test method for
 162      * {@link eu.etaxonomy.cdm.api.service.TaxonServiceImpl#findTaxaAndNames(eu.etaxonomy.cdm.api.service.config.IFindTaxaAndNamesConfigurator)}
 163      * .
 164      */
 165     @Test
 166     @DataSet
 167     public final void testFindTaxaAndNames() {
 168
 169         // pass 1
 170         IFindTaxaAndNamesConfigurator<?> configurator = new FindTaxaAndNamesConfiguratorImpl();
 171         configurator.setTitleSearchString("Abies*");
 172         configurator.setMatchMode(MatchMode.BEGINNING);
 173         configurator.setDoTaxa(true);
 174         configurator.setDoSynonyms(true);
 175         configurator.setDoNamesWithoutTaxa(true);
 176         configurator.setDoTaxaByCommonNames(true);
 177
 178         Pager<IdentifiableEntity> pager = taxonService.findTaxaAndNames(configurator);
 179         List<IdentifiableEntity> list = pager.getRecords();
 180
 181         if (logger.isDebugEnabled()) {
 182             for (int i = 0; i < list.size(); i++) {
 183                 String nameCache = "";
 184                 if (list.get(i) instanceof NonViralName) {
 185                     nameCache = ((NonViralName<?>) list.get(i)).getNameCache();
 186                 } else if (list.get(i) instanceof TaxonBase) {
 187                     TaxonNameBase<?,?> taxonNameBase = ((TaxonBase) list.get(i)).getName();
 188                     nameCache = HibernateProxyHelper.deproxy(taxonNameBase, NonViralName.class).getNameCache();
 189                 } else {
 190                 }
 191                 logger.debug(list.get(i).getClass() + "(" + i + ")" + ": Name Cache = " + nameCache + ", Title Cache = "
 192                         + list.get(i).getTitleCache());
 193             }
 194         }
 195
 196         logger.debug("number of taxa: " + list.size());
 197         assertEquals(8, list.size());
 198
 199         // pass 2
 200 //        configurator.setDoTaxaByCommonNames(false);
 201 //        configurator.setDoMisappliedNames(true);
 202 //        configurator.setClassification(classificationService.load(UUID.fromString(CLASSIFICATION_UUID)));
 203 //        pager = taxonService.findTaxaAndNames(configurator);
 204 //        list = pager.getRecords();
 205 //        assertEquals(0, list.size());
 206
 207     }
 208
 209     /**
 210      * Test method for
 211      * {@link eu.etaxonomy.cdm.api.service.TaxonServiceImpl#searchTaxaByName(java.lang.String, eu.etaxonomy.cdm.model.reference.Reference)}
 212      * .
 213      */
 214     @Test
 215     @DataSet
 216     public final void testSearchTaxaByName() {
 217         logger.warn("testSearchTaxaByName not yet implemented"); // TODO
 218     }
 219
 220     @SuppressWarnings("rawtypes")
 221     @Test
 222     @DataSet
 223     public final void testPurgeIndex() throws CorruptIndexException, IOException, ParseException {
 224
 225         refreshLuceneIndex();
 226
 227         Pager<SearchResult<TaxonBase>> pager;
 228
 229         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 7
 230         Assert.assertEquals("Expecting 8 entities", Integer.valueOf(8), pager.getCount());
 231
 232         indexer.purge(null);
 233         commitAndStartNewTransaction(null);
 234
 235         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 0
 236         Assert.assertEquals("Expecting no entities since the index has been purged", Integer.valueOf(0), pager.getCount());
 237     }
 238
 239     @SuppressWarnings("rawtypes")
 240     @Test
 241     @DataSet
 242     public final void testFindByDescriptionElementFullText_CommonName() throws CorruptIndexException, IOException,
 243             ParseException {
 244
 245         refreshLuceneIndex();
 246
 247         Pager<SearchResult<TaxonBase>> pager;
 248
 249         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null, null,
 250                 false, null, null, null, null);
 251         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", Integer.valueOf(1),
 252                 pager.getCount());
 253
 254         // the description containing the Nulltanne has no taxon attached,
 255         // taxon.id = null
 256         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Nulltanne", null, null, null,
 257                 false, null, null, null, null);
 258         Assert.assertEquals("Expecting no entity when searching for 'Nulltanne' ", Integer.valueOf(0), pager.getCount());
 259
 260         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 261                 Arrays.asList(new Language[] { Language.GERMAN() }), false, null, null, null, null);
 262         Assert.assertEquals("Expecting one entity when searching in German", Integer.valueOf(1), pager.getCount());
 263
 264         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"tanne", null, null,
 265                 Arrays.asList(new Language[] { Language.RUSSIAN() }), false, null, null, null, null);
 266         Assert.assertEquals("Expecting no entity when searching in Russian", Integer.valueOf(0), pager.getCount());
 267
 268     }
 269
 270     @SuppressWarnings("rawtypes")
 271     @Test
 272     @DataSet
 273     public final void testFindByDescriptionElementFullText_Distribution() throws CorruptIndexException, IOException, ParseException {
 274
 275         refreshLuceneIndex();
 276
 277         Pager<SearchResult<TaxonBase>> pager;
 278         // by Area
 279         pager = taxonService.findByDescriptionElementFullText(null, "Canada", null, null, null, false, null, null, null, null);
 280         Assert.assertEquals("Expecting one entity when searching for arae 'Canada'", Integer.valueOf(1), pager.getCount());
 281         // by Status
 282         pager = taxonService.findByDescriptionElementFullText(null, "present", null, null, null, false, null, null, null, null);
 283         Assert.assertEquals("Expecting one entity when searching for status 'present'", Integer.valueOf(1), pager.getCount());
 284     }
 285
 286     @SuppressWarnings("rawtypes")
 287     @Test
 288     @DataSet
 289     public final void testFindByDescriptionElementFullText_wildcard() throws CorruptIndexException, IOException, ParseException {
 290
 291         refreshLuceneIndex();
 292
 293         Pager<SearchResult<TaxonBase>> pager;
 294
 295         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
 296         Assert.assertEquals("Expecting one entity when searching for CommonTaxonName", Integer.valueOf(1), pager.getCount());
 297     }
 298
 299     /**
 300      * Regression test for #3113 (hibernate search: wildcard query can cause BooleanQuery$TooManyClauses: maxClauseCount is set to 1024)
 301      *
 302      * @throws CorruptIndexException
 303      * @throws IOException
 304      * @throws ParseException
 305      */
 306     @SuppressWarnings("rawtypes")
 307     @Test
 308     @DataSet
 309     public final void testFindByDescriptionElementFullText_TooManyClauses() throws CorruptIndexException, IOException, ParseException {
 310
 311         // generate 1024 terms to reproduce the bug
 312         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 313         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 314         while(uniqueRandomStrs.size() < 1024){
 315             uniqueRandomStrs.add(RandomStringUtils.random(10, true, false));
 316         }
 317         for(String rndStr: uniqueRandomStrs){
 318             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 319         }
 320         descriptionService.saveOrUpdate(description);
 321         commitAndStartNewTransaction(null);
 322
 323         refreshLuceneIndex();
 324
 325         Pager<SearchResult<TaxonBase>> pager;
 326
 327         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, null, null, null, null);
 328         Assert.assertEquals("Expecting all 1024 entities grouped into one SearchResult item when searching for Rot*", 1, pager.getCount().intValue());
 329     }
 330
 331     /**
 332      * Regression test for #3116 (fulltext search: always only one page of results)
 333      *
 334      * @throws CorruptIndexException
 335      * @throws IOException
 336      * @throws ParseException
 337      */
 338     @SuppressWarnings("rawtypes")
 339     @Test
 340     @DataSet
 341     public final void testFullText_Paging() throws CorruptIndexException, IOException, ParseException {
 342
 343         Reference sec = ReferenceFactory.newDatabase();
 344         referenceService.save(sec);
 345
 346         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 347         int numOfItems = 100;
 348         while(uniqueRandomStrs.size() < numOfItems){
 349             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 350         }
 351
 352         for(String rndStr: uniqueRandomStrs){
 353
 354             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(Rank.SERIES()), sec);
 355             taxon.setTitleCache("Tax" + rndStr, true);
 356             taxonService.save(taxon);
 357
 358             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 359             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 360             descriptionService.saveOrUpdate(description);
 361         }
 362
 363         commitAndStartNewTransaction(new String[]{"TAXONBASE"});
 364         refreshLuceneIndex();
 365
 366         int pageSize = 10;
 367
 368         Pager<SearchResult<TaxonBase>> pager;
 369
 370         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, null, null, null);
 371         Assert.assertEquals("unexpeted number of pages", Integer.valueOf(numOfItems / pageSize), pager.getPagesAvailable());
 372         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 9, null, null);
 373         Assert.assertNotNull("last page must have records", pager.getRecords());
 374         Assert.assertNotNull("last item on last page must exist", pager.getRecords().get(0));
 375         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, false, pageSize, 10, null, null);
 376         Assert.assertNotNull("last page + 1 must not have any records", pager.getRecords());
 377     }
 378
 379     /**
 380      * test for max score and sort by score of hit groups
 381      * with all matches per taxon in a single TextData  element
 382      * see {@link #testFullText_ScoreAndOrder_2()} for the complement
 383      * test with matches in multiple TextData per taxon
 384      *
 385      * @throws CorruptIndexException
 386      * @throws IOException
 387      * @throws ParseException
 388      */
 389     @SuppressWarnings("rawtypes")
 390     @Test
 391     @DataSet
 392     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 393     public final void testFullText_ScoreAndOrder_1() throws CorruptIndexException, IOException, ParseException {
 394
 395         int numOfTaxa = 3;
 396
 397         UUID[] taxonUuids = new UUID[numOfTaxa];
 398         StringBuilder text = new StringBuilder();
 399
 400         for(int i = 0; i < numOfTaxa; i++){
 401
 402             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(null), null);
 403             taxon.setTitleCache("Taxon_" + i, true);
 404             taxonUuids[i] = taxon.getUuid();
 405             taxonService.save(taxon);
 406
 407             text.append(" ").append("Rot");
 408             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 409             description.addElement(TextData.NewInstance(text.toString(), Language.DEFAULT(), null));
 410             descriptionService.saveOrUpdate(description);
 411         }
 412
 413         commitAndStartNewTransaction(null);
 414         refreshLuceneIndex();
 415
 416         Pager<SearchResult<TaxonBase>> pager;
 417
 418         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 419         for(int i = 0; i < numOfTaxa; i++){
 420             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 421         }
 422         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 423     }
 424
 425     /**
 426      * test for max score and sort by score of hit groups
 427      * with all matches per taxon in a multiple TextData elements
 428      * see {@link #testFullText_ScoreAndOrder_1()} for the complement
 429      * test with matches in a single TextData per taxon
 430      *
 431      * @throws CorruptIndexException
 432      * @throws IOException
 433      * @throws ParseException
 434      */
 435     @SuppressWarnings("rawtypes")
 436     @Test
 437     @DataSet
 438     @Ignore // test fails, maybe the assumptions made here are not compatible with the lucene scoring mechanism see http://lucene.apache.org/core/3_6_1/scoring.html
 439     public final void testFullText_ScoreAndOrder_2() throws CorruptIndexException, IOException, ParseException {
 440
 441         int numOfTaxa = 3;
 442
 443         UUID[] taxonUuids = new UUID[numOfTaxa];
 444
 445         for(int i = 0; i < numOfTaxa; i++){
 446
 447             Taxon taxon = Taxon.NewInstance(BotanicalName.NewInstance(null), null);
 448             taxon.setTitleCache("Taxon_" + i, true);
 449             taxonUuids[i] = taxon.getUuid();
 450             taxonService.save(taxon);
 451
 452             TaxonDescription description = TaxonDescription.NewInstance(taxon);
 453             for(int k = 0; k < i; k++){
 454                 description.addElement(TextData.NewInstance("Rot", Language.DEFAULT(), null));
 455             }
 456             descriptionService.saveOrUpdate(description);
 457         }
 458
 459         commitAndStartNewTransaction(null);
 460         refreshLuceneIndex();
 461
 462         Pager<SearchResult<TaxonBase>> pager;
 463
 464         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Rot", null, null, null, false, null, null, null, null);
 465         for(int i = 0; i < numOfTaxa; i++){
 466             Assert.assertEquals("taxa should be orderd by relevance (= score)", taxonUuids[numOfTaxa - i - 1], pager.getRecords().get(i).getEntity().getUuid());
 467         }
 468         Assert.assertEquals("max score should be equal to the score of the first element", pager.getRecords().get(0).getMaxScore(), pager.getRecords().get(0).getScore(), 0);
 469     }
 470
 471
 472     /**
 473      * @throws CorruptIndexException
 474      * @throws IOException
 475      * @throws ParseException
 476      * @throws LuceneMultiSearchException
 477      */
 478     @Test
 479     @DataSet
 480     public final void testFullText_Grouping() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 481
 482         TaxonDescription description = (TaxonDescription) descriptionService.find(UUID.fromString(D_ABIES_ALBA_UUID));
 483         Set<String> uniqueRandomStrs = new HashSet<String>(1024);
 484         int numOfItems = 100;
 485         while(uniqueRandomStrs.size() < numOfItems){
 486             uniqueRandomStrs.add(RandomStringUtils.random(5, true, false));
 487         }
 488         for(String rndStr: uniqueRandomStrs){
 489             description.addElement(CommonTaxonName.NewInstance("Rot" + rndStr, Language.DEFAULT()));
 490         }
 491         descriptionService.saveOrUpdate(description);
 492
 493         commitAndStartNewTransaction(new String[]{"DESCRIPTIONELEMENTBASE"});
 494
 495         refreshLuceneIndex();
 496
 497         int pageSize = 10;
 498
 499         Pager<SearchResult<TaxonBase>> pager;
 500         boolean highlightFragments = true;
 501
 502         // test with findByDescriptionElementFullText
 503         pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Rot*", null, null, null, highlightFragments, pageSize, null, null, null);
 504         logSearchResults(pager, Level.DEBUG, null);
 505         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 506         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 507         Map<String, String[]> highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 508         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 509         int maxDocsPerGroup = 10;
 510         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 511
 512         // test with findByEverythingFullText
 513         pager = taxonService.findByEverythingFullText( "Rot*", null, null, highlightFragments, pageSize, null, null, null);
 514         logSearchResults(pager, Level.DEBUG, null);
 515         Assert.assertEquals("All matches should be grouped into a single SearchResult element", 1, pager.getRecords().size());
 516         Assert.assertEquals("The count property of the pager must be set correctly", 1, pager.getCount().intValue());
 517         highlightMap = pager.getRecords().get(0).getFieldHighlightMap();
 518         // maxDocsPerGroup is defined in LuceneSearch and defaults to 10
 519         maxDocsPerGroup = 10;
 520         Assert.assertEquals("expecting 10 highlighted fragments of field 'name'", maxDocsPerGroup, highlightMap.get("name").length);
 521
 522     }
 523
 524     @SuppressWarnings("rawtypes")
 525     @Test
 526     @DataSet
 527     public final void testFindByDescriptionElementFullText_TextData() throws CorruptIndexException, IOException, ParseException {
 528
 529         refreshLuceneIndex();
 530
 531         Pager<SearchResult<TaxonBase>> pager;
 532         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, false, null, null, null, null);
 533         logSearchResults(pager, Level.DEBUG, null);
 534         Assert.assertEquals("Expecting one entity when searching for any TextData", Integer.valueOf(1), pager.getCount());
 535         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 536         Assert.assertTrue("Expecting two docs, one for RUSSIAN and one for GERMAN", pager.getRecords().get(0).getDocs().size() == 2);
 537         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 538
 539
 540         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, null, null, false, null, null, null, null);
 541         Assert.assertEquals("Expecting one entity when searching for any type", Integer.valueOf(1), pager.getCount());
 542
 543         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.UNKNOWN()}), null, false, null, null, null, null);
 544         Assert.assertEquals("Expecting one entity when searching for any type and for Feature DESCRIPTION", Integer.valueOf(1), pager.getCount());
 545
 546         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER()}), null, false, null, null, null, null);
 547         Assert.assertEquals("Expecting no entity when searching for any type and for Feature CHROMOSOME_NUMBER", Integer.valueOf(0), pager.getCount());
 548
 549         pager = taxonService.findByDescriptionElementFullText(null, "Abies", null, Arrays.asList(new Feature[]{Feature.CHROMOSOME_NUMBER(), Feature.UNKNOWN()}), null, false, null, null, null, null);
 550         Assert.assertEquals("Expecting no entity when searching for any type and for Feature DESCRIPTION or CHROMOSOME_NUMBER", Integer.valueOf(1), pager.getCount());
 551
 552         pager = taxonService.findByDescriptionElementFullText(Distribution.class, "Abies", null, null, null, false, null, null, null, null);
 553         Assert.assertEquals("Expecting no entity when searching for Distribution", Integer.valueOf(0), pager.getCount());
 554
 555         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{}), false, null, null, null, null);
 556         Assert.assertEquals("Expecting one entity", Integer.valueOf(1), pager.getCount());
 557         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 558
 559         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.RUSSIAN()}), false, null, null, null, null);
 560         Assert.assertEquals("Expecting one entity", Integer.valueOf(1), pager.getCount());
 561         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 562
 563         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Бальзам", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 564         Assert.assertEquals("Expecting no entity", Integer.valueOf(0), pager.getCount());
 565
 566         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 567         Assert.assertEquals("Expecting one entity", Integer.valueOf(1), pager.getCount());
 568         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 569     }
 570
 571     @SuppressWarnings("rawtypes")
 572     @Test
 573     @DataSet
 574     public final void testFindByDescriptionElementFullText_MultipleWords() throws CorruptIndexException, IOException, ParseException {
 575
 576         refreshLuceneIndex();
 577
 578         // Pflanzenart aus der Gattung der Tannen
 579
 580         Pager<SearchResult<TaxonBase>> pager;
 581         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, false, null, null, null, null);
 582         Assert.assertEquals("OR search : Expecting one entity", Integer.valueOf(1), pager.getCount());
 583
 584         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Wespen", null, null, null, false, null, null, null, null);
 585         Assert.assertEquals("OR search : Expecting one entity", Integer.valueOf(1), pager.getCount());
 586
 587         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, false, null, null, null, null);
 588         Assert.assertEquals("AND search : Expecting one entity", Integer.valueOf(1), pager.getCount());
 589
 590         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Wespen", null, null, null, false, null, null, null, null);
 591         Assert.assertEquals("AND search : Expecting no entity", Integer.valueOf(0), pager.getCount());
 592
 593         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, false, null, null, null, null);
 594         Assert.assertEquals("Phrase search : Expecting one entity", Integer.valueOf(1), pager.getCount());
 595
 596         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Wespen\"", null, null, null, false, null, null, null, null);
 597         Assert.assertEquals("Phrase search : Expecting one entity", Integer.valueOf(0), pager.getCount());
 598
 599
 600     }
 601
 602
 603     @SuppressWarnings("rawtypes")
 604     @Test
 605     @DataSet
 606     public final void testFindByDescriptionElementFullText_modify_DescriptionElement() throws CorruptIndexException, IOException, ParseException {
 607
 608         refreshLuceneIndex();
 609
 610         Pager<SearchResult<TaxonBase>> pager;
 611         //
 612         // modify the DescriptionElement
 613         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 614         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 615         Document indexDocument = pager.getRecords().get(0).getDocs().iterator().next();
 616         String[] descriptionElementUuidStr = indexDocument.getValues("uuid");
 617         String[] inDescriptionUuidStr = indexDocument.getValues("inDescription.uuid");
 618         // is only one uuid!
 619         DescriptionElementBase textData = descriptionService.getDescriptionElementByUuid(UUID.fromString(descriptionElementUuidStr[0]));
 620
 621         ((TextData)textData).removeText(Language.GERMAN());
 622         ((TextData)textData).putText(Language.SPANISH_CASTILIAN(), "abeto balsámico");
 623
 624         descriptionService.saveDescriptionElement(textData);
 625         commitAndStartNewTransaction(null);
 626 //        printDataSet(System.out, new String[] {
 627 //                "DESCRIPTIONELEMENTBASE", "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING" }
 628 //        );
 629
 630         //
 631         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN(), Language.RUSSIAN()}), false, null, null, null, null);
 632         Assert.assertEquals("The german 'Balsam-Tanne' TextData should no longer be indexed", Integer.valueOf(0), pager.getCount());
 633         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 634         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto balsámico'", Integer.valueOf(1), pager.getCount());
 635         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsámico", null, null, null, false, null, null, null, null);
 636         Assert.assertEquals("expecting to find the SPANISH_CASTILIAN 'abeto balsámico'", Integer.valueOf(1), pager.getCount());
 637
 638         //
 639         // modify the DescriptionElement via the Description object
 640         DescriptionBase<?> description = descriptionService.find(UUID.fromString(inDescriptionUuidStr[0]));
 641         Set<DescriptionElementBase> elements = description.getElements();
 642         for( DescriptionElementBase elm : elements){
 643             if(elm.getUuid().toString().equals(descriptionElementUuidStr[0])){
 644                 ((TextData)elm).removeText(Language.SPANISH_CASTILIAN());
 645                 ((TextData)elm).putText(Language.POLISH(), "Jodła balsamiczna");
 646             }
 647         }
 648         descriptionService.saveOrUpdate(description);
 649         commitAndStartNewTransaction(null);
 650         pager = taxonService.findByDescriptionElementFullText(TextData.class, "abeto", null, null, Arrays.asList(new Language[]{Language.SPANISH_CASTILIAN()}), false, null, null, null, null);
 651         Assert.assertEquals("The spanish 'abeto balsámico' TextData should no longer be indexed", Integer.valueOf(0), pager.getCount());
 652         pager = taxonService.findByDescriptionElementFullText(TextData.class, "balsamiczna", null, null, Arrays.asList(new Language[]{Language.POLISH()}), false, null, null, null, null);
 653         Assert.assertEquals("expecting to find the POLISH 'Jodła balsamiczna'", Integer.valueOf(1), pager.getCount());
 654     }
 655
 656     @SuppressWarnings("rawtypes")
 657     @Test
 658     @DataSet
 659     public final void testFindByDescriptionElementFullText_modify_Taxon() throws CorruptIndexException, IOException, ParseException {
 660
 661         refreshLuceneIndex();
 662
 663         Pager<SearchResult<TaxonBase>> pager;
 664         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
 665         TaxonDescription d_abies_balsamea = (TaxonDescription)descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 666
 667         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 668         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne'", Integer.valueOf(1), pager.getCount());
 669
 670         // exchange the Taxon with another one via the Taxon object
 671         // 1.) remove existing description:
 672         t_abies_balsamea.removeDescription(d_abies_balsamea);
 673
 674         taxonService.saveOrUpdate(t_abies_balsamea);
 675         commitAndStartNewTransaction(null);
 676
 677
 678         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 679         Assert.assertEquals("'Balsam-Tanne' should no longer be found", Integer.valueOf(0), pager.getCount());
 680
 681         // 2.) create new description and add to taxon:
 682         TaxonDescription d_abies_balsamea_new = TaxonDescription.NewInstance();
 683         d_abies_balsamea_new
 684                 .addElement(TextData
 685                         .NewInstance(
 686                                 "Die Balsamtanne ist mit bis zu 30 m Höhe ein mittelgro"+UTF8.SHARP_S+"er Baum und kann bis zu 200 Jahre alt werden",
 687                                 Language.GERMAN(), null));
 688         t_abies_balsamea.addDescription(d_abies_balsamea_new);
 689         // set authorshipCache to null to avoid validation exception,
 690         // this is maybe not needed in future,  see ticket #3344
 691         BotanicalName abies_balsamea = HibernateProxyHelper.deproxy(t_abies_balsamea.getName(), BotanicalName.class);
 692         abies_balsamea.setAuthorshipCache(null);
 693         taxonService.saveOrUpdate(t_abies_balsamea);
 694         commitAndStartNewTransaction(null);
 695
 696         printDataSet(System.out, new String[] {
 697                 "DESCRIPTIONBASE"
 698         });
 699
 700         pager = taxonService.findByDescriptionElementFullText(TextData.class, "mittelgro"+UTF8.SHARP_S+"er Baum", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 701         Assert.assertEquals("the taxon should be found via the new Description", Integer.valueOf(1), pager.getCount());
 702     }
 703
 704     @SuppressWarnings("rawtypes")
 705     @Test
 706     @DataSet
 707     public final void testFindByDescriptionElementFullText_modify_Classification() throws CorruptIndexException, IOException, ParseException {
 708
 709         refreshLuceneIndex();
 710
 711         Pager<SearchResult<TaxonBase>> pager;
 712
 713         // put taxon into other classification, new taxon node
 714         Classification classification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 715         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 716
 717         // TODO: why is the test failing when the childNode is already retrieved here, and not after the following four lines?
 718         //TaxonNode childNode = classification.getChildNodes().iterator().next();
 719
 720         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", null, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 721         Assert.assertEquals("expecting to find the GERMAN 'Balsam-Tanne' even if filtering by classification", Integer.valueOf(1), pager.getCount());
 722         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 723         Assert.assertEquals("GERMAN 'Balsam-Tanne' should NOT be found in other classification", Integer.valueOf(0), pager.getCount());
 724
 725         // check for the right taxon node
 726         TaxonNode childNode = classification.getChildNodes().iterator().next();
 727         Assert.assertEquals("expecting Abies balsamea sec.", childNode.getTaxon().getUuid().toString(), ABIES_BALSAMEA_UUID);
 728         Assert.assertEquals("expecting default classification", childNode.getClassification().getUuid().toString(), CLASSIFICATION_UUID);
 729
 730         // moving the taxon around
 731         alternateClassification.addChildNode(childNode, null, null);
 732         classificationService.saveOrUpdate(alternateClassification);
 733         commitAndStartNewTransaction(null);
 734
 735 //        printDataSet(System.out, new String[] {
 736 //            "TAXONBASE", "TAXONNODE", "CLASSIFICATION"
 737 //        });
 738
 739         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", alternateClassification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 740         Assert.assertEquals("GERMAN 'Balsam-Tanne' should now be found in other classification", Integer.valueOf(1), pager.getCount());
 741
 742         classification.getChildNodes().clear();
 743         classificationService.saveOrUpdate(classification);
 744         commitAndStartNewTransaction(null);
 745
 746         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Balsam-Tanne", classification, null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
 747         Assert.assertEquals("Now the GERMAN 'Balsam-Tanne' should NOT be found in original classification", Integer.valueOf(0), pager.getCount());
 748
 749     }
 750
 751     @SuppressWarnings("rawtypes")
 752     @Test
 753     @DataSet
 754     public final void testFindByDescriptionElementFullText_CategoricalData() throws CorruptIndexException, IOException, ParseException {
 755
 756         // add CategoricalData
 757         DescriptionBase d_abies_balsamea = descriptionService.find(UUID.fromString(D_ABIES_BALSAMEA_UUID));
 758         // Categorical data
 759         CategoricalData cdata = CategoricalData.NewInstance();
 760         cdata.setFeature(Feature.DESCRIPTION());
 761         State state = State.NewInstance("green", "green", "gn");
 762
 763         StateData statedata = StateData.NewInstance(state);
 764         statedata.putModifyingText(Language.ENGLISH(), "always, even during winter");
 765         cdata.addStateData(statedata);
 766         d_abies_balsamea.addElement(cdata);
 767
 768         UUID termUUID = termService.save(state);
 769         descriptionService.save(d_abies_balsamea);
 770
 771         commitAndStartNewTransaction(null);
 772
 773 //        printDataSet(System.out, new String[] {
 774 //                 "STATEDATA", "STATEDATA_DEFINEDTERMBASE", "STATEDATA_LANGUAGESTRING", "LANGUAGESTRING"});
 775
 776         refreshLuceneIndex();
 777
 778         Pager<SearchResult<TaxonBase>> pager;
 779         pager = taxonService.findByDescriptionElementFullText(CategoricalData.class, "green", null, null, null, false, null, null, null, null);
 780         Assert.assertEquals("Expecting one entity", Integer.valueOf(1), pager.getCount());
 781         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getEntity().getTitleCache());
 782         Assert.assertTrue("Expecting only one doc", pager.getRecords().get(0).getDocs().size() == 1);
 783         Assert.assertEquals("Abies balsamea sec. Kohlbecker, A., Testcase standart views, 2013", pager.getRecords().get(0).getDocs().iterator().next().get("inDescription.taxon.titleCache"));
 784
 785
 786         //TODO modify the StateData
 787         TaxonBase taxon = pager.getRecords().get(0).getEntity();
 788
 789         String newName = "Quercus robur";
 790         taxon.setTitleCache(newName + " sec. ", true);
 791
 792         taxonService.saveOrUpdate(taxon);
 793         commitAndStartNewTransaction(null);
 794
 795         taxon = taxonService.find(taxon.getUuid());
 796         Assert.assertEquals(newName + " sec. ", taxon.getTitleCache());
 797         DefinedTermBase term = termService.find(termUUID);
 798         try {
 799                 termService.delete(term);
 800         }catch(ReferencedObjectUndeletableException e){
 801                 Assert.fail();
 802         }
 803     }
 804
 805     @SuppressWarnings("rawtypes")
 806     @Test
 807     @DataSet
 808     public final void testFindByDescriptionElementFullText_Highlighting() throws CorruptIndexException, IOException, ParseException {
 809
 810         refreshLuceneIndex();
 811
 812         Pager<SearchResult<TaxonBase>> pager;
 813         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Abies", null, null, null, true, null, null, null, null);
 814         Assert.assertEquals("Expecting one entity when searching for any TextData", Integer.valueOf(1), pager.getCount());
 815         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 816         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
 817         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 818         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Abies</B>"));
 819
 820         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Pflanzenart Tannen", null, null, null, true, null, null, null, null);
 821         searchResult = pager.getRecords().get(0);
 822         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 823         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 824         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") || fragments[0].contains("<B>Tannen</B>"));
 825
 826         pager = taxonService.findByDescriptionElementFullText(TextData.class, "+Pflanzenart +Tannen", null, null, null, true, null, null, null, null);
 827         searchResult = pager.getRecords().get(0);
 828         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 829         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 830         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B>") && fragments[0].contains("<B>Tannen</B>"));
 831
 832         pager = taxonService.findByDescriptionElementFullText(TextData.class, "\"Pflanzenart aus der Gattung der Tannen\"", null, null, null, true, null, null, null, null);
 833         searchResult = pager.getRecords().get(0);
 834         Assert.assertTrue("Phrase search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 835         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 836         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Pflanzenart</B> <B>aus</B> <B>der</B> <B>Gattung</B> <B>der</B> <B>Tannen</B>"));
 837
 838         pager = taxonService.findByDescriptionElementFullText(TextData.class, "Gatt*", null, null, null, true, null, null, null, null);
 839         searchResult = pager.getRecords().get(0);
 840         Assert.assertTrue("Wildcard search : Expecting at least one item in highlighted fragments", searchResult.getFieldHighlightMap().size() > 0);
 841         fragments = searchResult.getFieldHighlightMap().values().iterator().next();
 842         Assert.assertTrue("first fragments should contains serch term", fragments[0].contains("<B>Gatt"));
 843     }
 844
 845
 846     @Test
 847     @DataSet
 848     public final void testFindByFullText() throws CorruptIndexException, IOException, ParseException {
 849
 850         refreshLuceneIndex();
 851
 852         Classification europeanAbiesClassification = classificationService.find(UUID.fromString(CLASSIFICATION_UUID));
 853
 854         Pager<SearchResult<TaxonBase>> pager;
 855
 856         pager = taxonService.findByFullText(null, "Abies", null, null, true, null, null, null, null); // --> 7
 857         logSearchResults(pager, Level.DEBUG, null);
 858         Assert.assertEquals("Expecting 8 entities", Integer.valueOf(8), pager.getCount());
 859
 860         pager = taxonService.findByFullText(Taxon.class, "Abies", null, null, true, null, null, null, null); // --> 6
 861         Assert.assertEquals("Expecting 7 entities", Integer.valueOf(7), pager.getCount());
 862
 863         pager = taxonService.findByFullText(Synonym.class, "Abies", null, null, true, null, null, null, null); // --> 1
 864         Assert.assertEquals("Expecting 1 entity", Integer.valueOf(1), pager.getCount());
 865
 866         pager = taxonService.findByFullText(TaxonBase.class, "sec", null, null, true, null, null, null, null); // --> 7
 867         Assert.assertEquals("Expecting 8 entities", Integer.valueOf(8), pager.getCount());
 868
 869         pager = taxonService.findByFullText(null, "genus", null, null, true, null, null, null, null); // --> 1
 870         Assert.assertEquals("Expecting 1 entity", Integer.valueOf(1), pager.getCount());
 871
 872         pager = taxonService.findByFullText(Taxon.class, "subalpina", null, null, true, null, null, null, null); // --> 0
 873         Assert.assertEquals("Expecting 0 entities", Integer.valueOf(0), pager.getCount());
 874
 875         // synonym in classification ???
 876     }
 877
 878     @Test
 879     @DataSet
 880     public final void testPrepareByAreaSearch() throws IOException, ParseException {
 881
 882         List<PresenceAbsenceTermBase<?>> statusFilter = new ArrayList<PresenceAbsenceTermBase<?>>();
 883         List<NamedArea> areaFilter = new ArrayList<NamedArea>();
 884         areaFilter.add(germany);
 885         areaFilter.add(canada);
 886         areaFilter.add(russia);
 887
 888         Pager<SearchResult<TaxonBase>> pager = taxonService.findByDistribution(areaFilter, statusFilter, null, 20, 0, null, null);
 889         Assert.assertEquals("Expecting 2 entities", Integer.valueOf(2), Integer.valueOf(pager.getRecords().size()));
 890
 891     }
 892
 893     @Test
 894     @DataSet
 895     public final void testFindTaxaAndNamesByFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 896
 897         refreshLuceneIndex();
 898
 899         Pager<SearchResult<TaxonBase>> pager;
 900
 901         Classification alternateClassification = classificationService.find(UUID.fromString(CLASSIFICATION_ALT_UUID));
 902
 903
 904         pager = taxonService.findTaxaAndNamesByFullText(
 905                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 906                 "Abies", null, null, null, null, true, null, null, null, null);
 907 //        logPagerRecords(pager, Level.DEBUG);
 908         Assert.assertEquals("doTaxa & doSynonyms", Integer.valueOf(8), pager.getCount());
 909
 910         pager = taxonService.findTaxaAndNamesByFullText(
 911                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 912                 "Abies", null, null, null, null, true, null, null, null, null);
 913 //        logPagerRecords(pager, Level.DEBUG);
 914         Assert.assertEquals("all search modes", Integer.valueOf(8), pager.getCount());
 915
 916         pager = taxonService.findTaxaAndNamesByFullText(
 917                 EnumSet.allOf(TaxaAndNamesSearchMode.class),
 918                 "Abies", alternateClassification, null, null, null, true, null, null, null, null);
 919 //        logPagerRecords(pager, Level.DEBUG);
 920         Assert.assertEquals("all search modes, filtered by alternateClassification", Integer.valueOf(1), pager.getCount());
 921
 922         pager = taxonService.findTaxaAndNamesByFullText(
 923                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
 924                 "Abies", null, null, null, null, true, null, null, null, null);
 925         Assert.assertEquals("Expecting 1 entity", Integer.valueOf(1), pager.getCount());
 926         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
 927         Assert.assertEquals(Synonym.class, searchResult.getEntity().getClass());
 928         // Abies subalpina sec. Kohlbecker, A., Testcase standart views, 2013
 929
 930
 931         pager = taxonService.findTaxaAndNamesByFullText(
 932                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 933                 "Abies", null, null, null, null, true, null, null, null, null);
 934         Assert.assertEquals("Expecting 0 entity", Integer.valueOf(0), pager.getCount());
 935
 936
 937         pager = taxonService.findTaxaAndNamesByFullText(
 938                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
 939                 "Tanne", null, null, null, null, true, null, null, null, null);
 940         Assert.assertEquals("Expecting 1 entity", Integer.valueOf(1), Integer.valueOf(pager.getRecords().size()));
 941         Assert.assertEquals("Expecting 1 entity", Integer.valueOf(1), pager.getCount());
 942
 943         pager = taxonService.findTaxaAndNamesByFullText(
 944                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
 945                 "kawakamii", null, null, null, null, true, null, null, null, null);
 946         logSearchResults(pager, Level.DEBUG, null);
 947         Assert.assertEquals("Expecting 1 entity", Integer.valueOf(1), pager.getCount());
 948
 949     }
 950
 951     @Test
 952     @DataSet
 953     public final void testFindTaxaAndNamesByFullText_Sort() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
 954
 955         refreshLuceneIndex();
 956
 957         Pager<SearchResult<TaxonBase>> pager;
 958
 959         List<OrderHint> orderHints = new ArrayList<OrderHint>();
 960
 961         String[] docFields2log = new String[]{"id"};
 962
 963         // SortById
 964         orderHints.addAll(OrderHint.ORDER_BY_ID);
 965         pager = taxonService.findTaxaAndNamesByFullText(
 966                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
 967                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 968 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 969         int lastId = -1;
 970         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 971             if(lastId != -1){
 972                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
 973             }
 974             lastId = rs.getEntity().getId();
 975         }
 976
 977         orderHints.addAll(OrderHint.ORDER_BY_ID);
 978         pager = taxonService.findTaxaAndNamesByFullText(
 979                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 980                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 981 //        logSearchResults(pager, Level.DEBUG, docFields2log);
 982
 983         lastId = -1;
 984         for(SearchResult<TaxonBase> rs : pager.getRecords()){
 985             if(lastId != -1){
 986                 Assert.assertTrue("results not sorted by id", lastId < rs.getEntity().getId());
 987             }
 988             lastId = rs.getEntity().getId();
 989         }
 990
 991         // Sortby NOMENCLATURAL_SORT_ORDER TODO make assertions !!!
 992         orderHints.clear();
 993         orderHints.addAll(OrderHint.NOMENCLATURAL_SORT_ORDER);
 994         pager = taxonService.findTaxaAndNamesByFullText(
 995                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
 996                 "Abies", null, null, null, null, true, null, null, orderHints, null);
 997         logSearchResults(pager, Level.DEBUG, null);
 998
 999     }
1000
1001     @Test
1002     @DataSet
1003     public final void testFindTaxaAndNamesByFullText_AreaFilter() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1004
1005         refreshLuceneIndex();
1006
1007         Pager<SearchResult<TaxonBase>> pager;
1008
1009         Set<NamedArea> a_germany_canada_russia = new HashSet<NamedArea>();
1010         a_germany_canada_russia.add(germany);
1011         a_germany_canada_russia.add(canada);
1012         a_germany_canada_russia.add(russia);
1013
1014         Set<NamedArea> a_russia = new HashSet<NamedArea>();
1015         a_russia.add(russia);
1016
1017         Set<PresenceAbsenceTermBase<?>> present = new HashSet<PresenceAbsenceTermBase<?>>();
1018         present.add(PresenceTerm.PRESENT());
1019
1020         Set<PresenceAbsenceTermBase<?>> present_native = new HashSet<PresenceAbsenceTermBase<?>>();
1021         present_native.add(PresenceTerm.PRESENT());
1022         present_native.add(PresenceTerm.NATIVE());
1023
1024         Set<PresenceAbsenceTermBase<?>> absent = new HashSet<PresenceAbsenceTermBase<?>>();
1025         absent.add(AbsenceTerm.ABSENT());
1026
1027         pager = taxonService.findTaxaAndNamesByFullText(
1028                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa),
1029                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1030         logSearchResults(pager, Level.DEBUG, null);
1031         Assert.assertEquals("taxa with matching area filter", Integer.valueOf(2), pager.getCount());
1032
1033         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1034         pager = taxonService.findTaxaAndNamesByFullText(
1035                 EnumSet.of(TaxaAndNamesSearchMode.doSynonyms),
1036                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1037         Assert.assertEquals("synonyms with matching area filter", Integer.valueOf(1), pager.getCount());
1038
1039         pager = taxonService.findTaxaAndNamesByFullText(
1040                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1041                 "Abies", null, a_germany_canada_russia, null, null, true, null, null, null, null);
1042         logSearchResults(pager, Level.DEBUG, null);
1043         Assert.assertEquals("taxa and synonyms with matching area filter", Integer.valueOf(3), pager.getCount());
1044
1045         pager = taxonService.findTaxaAndNamesByFullText(
1046                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1047                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1048         Assert.assertEquals("taxa and synonyms with matching area & status filter 1", Integer.valueOf(3), pager.getCount());
1049
1050         pager = taxonService.findTaxaAndNamesByFullText(
1051                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1052                 "Abies", null, a_germany_canada_russia, present, null, true, null, null, null, null);
1053         Assert.assertEquals("taxa and synonyms with matching area & status filter 2", Integer.valueOf(2), pager.getCount());
1054
1055         pager = taxonService.findTaxaAndNamesByFullText(
1056                 EnumSet.of(TaxaAndNamesSearchMode.doTaxa, TaxaAndNamesSearchMode.doSynonyms),
1057                 "Abies", null, a_russia, present, null, true, null, null, null, null);
1058         Assert.assertEquals("taxa and synonyms with non matching area & status filter", Integer.valueOf(0), pager.getCount());
1059
1060         pager = taxonService.findTaxaAndNamesByFullText(
1061                 EnumSet.of(TaxaAndNamesSearchMode.doTaxaByCommonNames),
1062                 "Tanne", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1063         Assert.assertEquals("ByCommonNames with area filter", Integer.valueOf(1), pager.getCount());
1064
1065         // abies_kawakamii_sensu_komarov as misapplied name for t_abies_balsamea
1066         pager = taxonService.findTaxaAndNamesByFullText(
1067                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1068                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1069         Assert.assertEquals("misappliedNames with matching area & status filter", Integer.valueOf(1), pager.getCount());
1070
1071
1072         // 1. remove existing taxon relation
1073         Taxon t_abies_balsamea = (Taxon)taxonService.find(UUID.fromString(ABIES_BALSAMEA_UUID));
1074         Set<TaxonRelationship> relsTo = t_abies_balsamea.getRelationsToThisTaxon();
1075         Assert.assertEquals(Integer.valueOf(1), Integer.valueOf(relsTo.size()));
1076         TaxonRelationship taxonRelation = relsTo.iterator().next();
1077         t_abies_balsamea.removeTaxonRelation(taxonRelation);
1078         taxonService.saveOrUpdate(t_abies_balsamea);
1079         commitAndStartNewTransaction(null);
1080
1081         pager = taxonService.findTaxaAndNamesByFullText(
1082                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1083                 "Abies", null, a_germany_canada_russia, present_native, null, true, null, null, null, null);
1084         Assert.assertEquals("misappliedNames with matching area & status filter, should match nothing now", Integer.valueOf(0), pager.getCount());
1085
1086         // 2. now add abies_kawakamii_sensu_komarov as misapplied name for t_abies_alba and search for misapplications in russia: ABSENT
1087         Taxon t_abies_kawakamii_sensu_komarov = (Taxon)taxonService.find(UUID.fromString(D_ABIES_KAWAKAMII_SEC_KOMAROV_UUID));
1088         Taxon t_abies_alba = (Taxon)taxonService.find(UUID.fromString(ABIES_ALBA_UUID));
1089         t_abies_alba.addMisappliedName(t_abies_kawakamii_sensu_komarov, null, null);
1090         taxonService.update(t_abies_alba);
1091         commitAndStartNewTransaction(null);
1092
1093         pager = taxonService.findTaxaAndNamesByFullText(
1094                 EnumSet.of(TaxaAndNamesSearchMode.doMisappliedNames),
1095                 "Abies", null, a_germany_canada_russia, absent, null, true, null, null, null, null);
1096         Assert.assertEquals("misappliedNames with matching area & status filter, should find one", Integer.valueOf(1), pager.getCount());
1097
1098     }
1099
1100     /**
1101      * Regression test for #3119: fulltext search: Entity always null whatever search
1102      *
1103      * @throws CorruptIndexException
1104      * @throws IOException
1105      * @throws ParseException
1106      * @throws LuceneMultiSearchException
1107      */
1108     @Test
1109     @DataSet
1110     public final void testFindByEverythingFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1111
1112         refreshLuceneIndex();
1113
1114         Pager<SearchResult<TaxonBase>> pager;
1115
1116         // via Taxon
1117         pager = taxonService.findByEverythingFullText("Abies", null, null, true, null, null, null, null);
1118         logSearchResults(pager, Level.DEBUG, null);
1119         Assert.assertTrue("Expecting at least 7 entities for 'Abies'", pager.getCount() > 7);
1120         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1121         Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1122
1123         // via DescriptionElement
1124         pager = taxonService.findByEverythingFullText("present", null, null, true, null, null, null, null);
1125         Assert.assertEquals("Expecting one entity when searching for area 'present'", Integer.valueOf(1), pager.getCount());
1126         Assert.assertNotNull("Expecting entity", pager.getRecords().get(0).getEntity());
1127         Assert.assertEquals("Expecting Taxon entity", Taxon.class, pager.getRecords().get(0).getEntity().getClass());
1128         Assert.assertEquals("Expecting Taxon ", ABIES_BALSAMEA_UUID, pager.getRecords().get(0).getEntity().getUuid().toString());
1129
1130     }
1131
1132
1133     @Test
1134     @DataSet
1135     public final void findByEveryThingFullText() throws CorruptIndexException, IOException, ParseException, LuceneMultiSearchException {
1136
1137         refreshLuceneIndex();
1138
1139         Pager<SearchResult<TaxonBase>> pager;
1140
1141         pager = taxonService.findByEverythingFullText("genus", null, null,  false, null, null, null, null); // --> 1
1142         Assert.assertEquals("Expecting 1 entity", Integer.valueOf(1), pager.getCount());
1143
1144         //FIXME FAILS: abies balamea is returned twice, see also testFullText_Grouping()
1145         pager = taxonService.findByEverythingFullText("Balsam-Tanne", null, Arrays.asList(new Language[]{Language.GERMAN()}), false, null, null, null, null);
1146         logSearchResults(pager, Level.DEBUG, null);
1147         Assert.assertEquals("expecting to find the Abies balsamea via the GERMAN DescriptionElements", Integer.valueOf(1), pager.getCount());
1148
1149         pager = taxonService.findByEverythingFullText("Abies", null, null, true, null, null, null, null);
1150         Assert.assertEquals("Expecting 8 entities", Integer.valueOf(8), pager.getCount());
1151         SearchResult<TaxonBase> searchResult = pager.getRecords().get(0);
1152         Assert.assertTrue("the map of highlighted fragments should contain at least one item", searchResult.getFieldHighlightMap().size() > 0);
1153         String[] fragments = searchResult.getFieldHighlightMap().values().iterator().next();
1154         Assert.assertTrue("first fragments should contains serch term", fragments[0].toLowerCase().contains("<b>abies</b>"));
1155     }
1156
1157 //    @SuppressWarnings("rawtypes")
1158 //    @Test
1159 //    @DataSet
1160 //    public final void benchmarkFindTaxaAndNamesHql() throws CorruptIndexException, IOException, ParseException {
1161 //
1162 //        createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1163 //
1164 //        IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1165 //        configurator.setTitleSearchString("Weiß%");
1166 //        configurator.setMatchMode(MatchMode.BEGINNING);
1167 //        configurator.setDoTaxa(false);
1168 //        configurator.setDoSynonyms(false);
1169 //        configurator.setDoNamesWithoutTaxa(false);
1170 //        configurator.setDoTaxaByCommonNames(true);
1171 //
1172 //        Pager<IdentifiableEntity> pager;
1173 //
1174 //        long startMillis = System.currentTimeMillis();
1175 //        for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1176 //            pager = taxonService.findTaxaAndNames(configurator);
1177 //            if (logger.isDebugEnabled()) {
1178 //                logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1179 //            }
1180 //        }
1181 //        double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1182 //        logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1183 //    }
1184
1185     @SuppressWarnings("rawtypes")
1186     @Test
1187     @DataSet
1188     public final void benchmarkFindByCommonNameHql() throws CorruptIndexException, IOException, ParseException {
1189
1190 //        printDataSet(System.err, new String[] { "TaxonBase" });
1191
1192         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1193
1194         IFindTaxaAndNamesConfigurator configurator = new FindTaxaAndNamesConfiguratorImpl();
1195         configurator.setTitleSearchString("Wei"+UTF8.SHARP_S+"%");
1196         configurator.setMatchMode(MatchMode.BEGINNING);
1197         configurator.setDoTaxa(false);
1198         configurator.setDoSynonyms(false);
1199         configurator.setDoNamesWithoutTaxa(false);
1200         configurator.setDoTaxaByCommonNames(true);
1201
1202         Pager<IdentifiableEntity> pager;
1203
1204         long startMillis = System.currentTimeMillis();
1205         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1206             pager = taxonService.findTaxaAndNames(configurator);
1207             if (logger.isDebugEnabled()) {
1208                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getTitleCache());
1209             }
1210         }
1211         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1212         logger.info("Benchmark result - [find taxon by CommonName via HQL] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1213     }
1214
1215     @SuppressWarnings("rawtypes")
1216     @Test
1217     @DataSet
1218     public final void benchmarkFindByCommonNameLucene() throws CorruptIndexException, IOException, ParseException {
1219
1220         createRandomTaxonWithCommonName(NUM_OF_NEW_RADOM_ENTITIES);
1221
1222         refreshLuceneIndex();
1223
1224         Pager<SearchResult<TaxonBase>> pager;
1225
1226         long startMillis = System.currentTimeMillis();
1227         for (int indx = 0; indx < BENCHMARK_ROUNDS; indx++) {
1228             pager = taxonService.findByDescriptionElementFullText(CommonTaxonName.class, "Wei"+UTF8.SHARP_S+"*", null, null, null, false, null, null, null, null);
1229             if (logger.isDebugEnabled()) {
1230                 logger.debug("[" + indx + "]" + pager.getRecords().get(0).getEntity().getTitleCache());
1231             }
1232         }
1233         double duration = ((double) (System.currentTimeMillis() - startMillis)) / BENCHMARK_ROUNDS;
1234         logger.info("Benchmark result - [find taxon by CommonName via lucene] : " + duration + "ms (" + BENCHMARK_ROUNDS + " benchmark rounds )");
1235     }
1236
1237     /**
1238      * uncomment @Test annotation to create the dataset for this test
1239      */
1240 //    @Test
1241     @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="BlankDataSet.xml")
1242     public final void createDataSet() throws FileNotFoundException {
1243
1244         Classification europeanAbiesClassification = Classification.NewInstance("European Abies");
1245         europeanAbiesClassification.setUuid(UUID.fromString(CLASSIFICATION_UUID));
1246         classificationService.save(europeanAbiesClassification);
1247
1248         Classification alternativeClassification = Classification.NewInstance("Abies alternative");
1249         alternativeClassification.setUuid(UUID.fromString(CLASSIFICATION_ALT_UUID));
1250         classificationService.save(alternativeClassification);
1251
1252         Reference<?> sec = ReferenceFactory.newBook();
1253         sec.setTitleCache("Kohlbecker, A., Testcase standart views, 2013", true);
1254         Reference<?> sec_sensu = ReferenceFactory.newBook();
1255         sec_sensu.setTitleCache("Komarov, V. L., Flora SSSR 29", true);
1256         referenceService.save(sec);
1257         referenceService.save(sec_sensu);
1258
1259         BotanicalName n_abies = BotanicalName.NewInstance(Rank.GENUS());
1260         n_abies.setNameCache("Abies", true);
1261         Taxon t_abies = Taxon.NewInstance(n_abies, sec);
1262         taxonService.save(t_abies);
1263
1264         BotanicalName n_abies_alba = BotanicalName.NewInstance(Rank.SPECIES());
1265         n_abies_alba.setNameCache("Abies alba", true);
1266         Taxon t_abies_alba = Taxon.NewInstance(n_abies_alba, sec);
1267         t_abies_alba.setUuid(UUID.fromString(ABIES_ALBA_UUID));
1268         taxonService.save(t_abies_alba);
1269
1270         BotanicalName n_abies_subalpina = BotanicalName.NewInstance(Rank.SPECIES());
1271         n_abies_subalpina.setNameCache("Abies subalpina", true);
1272         Synonym s_abies_subalpina = Synonym.NewInstance(n_abies_subalpina, sec);
1273         taxonService.save(s_abies_subalpina);
1274
1275         BotanicalName n_abies_balsamea = BotanicalName.NewInstance(Rank.SPECIES());
1276         n_abies_balsamea.setNameCache("Abies balsamea", true);
1277         Taxon t_abies_balsamea = Taxon.NewInstance(n_abies_balsamea, sec);
1278         t_abies_balsamea.setUuid(UUID.fromString(ABIES_BALSAMEA_UUID));
1279         t_abies_balsamea.addSynonym(s_abies_subalpina, SynonymRelationshipType.SYNONYM_OF());
1280         taxonService.save(t_abies_balsamea);
1281
1282         BotanicalName n_abies_grandis = BotanicalName.NewInstance(Rank.SPECIES());
1283         n_abies_grandis.setNameCache("Abies grandis", true);
1284         Taxon t_abies_grandis = Taxon.NewInstance(n_abies_grandis, sec);
1285         taxonService.save(t_abies_grandis);
1286
1287         BotanicalName n_abies_kawakamii = BotanicalName.NewInstance(Rank.SPECIES());
1288         n_abies_kawakamii.setNameCache("Abies kawakamii", true);
1289         Taxon t_abies_kawakamii = Taxon.NewInstance(n_abies_kawakamii, sec);
1290         t_abies_kawakamii.getTitleCache();
1291         taxonService.save(t_abies_kawakamii);
1292
1293         // abies_kawakamii_sensu_komarov as missapplied name for t_abies_balsamea
1294         Taxon t_abies_kawakamii_sensu_komarov = Taxon.NewInstance(n_abies_kawakamii, sec_sensu);
1295         taxonService.save(t_abies_kawakamii_sensu_komarov);
1296         t_abies_kawakamii_sensu_komarov.addTaxonRelation(t_abies_balsamea, TaxonRelationshipType.MISAPPLIED_NAME_FOR(), null, null);
1297         taxonService.saveOrUpdate(t_abies_kawakamii_sensu_komarov);
1298
1299         BotanicalName n_abies_lasiocarpa = BotanicalName.NewInstance(Rank.SPECIES());
1300         n_abies_lasiocarpa.setNameCache("Abies lasiocarpa", true);
1301         Taxon t_abies_lasiocarpa = Taxon.NewInstance(n_abies_lasiocarpa, sec);
1302         taxonService.save(t_abies_lasiocarpa);
1303
1304         // add taxa to classifications
1305         europeanAbiesClassification.addChildTaxon(t_abies_balsamea, null, null);
1306         alternativeClassification.addChildTaxon(t_abies_lasiocarpa, null, null);
1307         classificationService.saveOrUpdate(europeanAbiesClassification);
1308         classificationService.saveOrUpdate(alternativeClassification);
1309
1310         //
1311         // Description
1312         //
1313         TaxonDescription d_abies_alba = TaxonDescription.NewInstance(t_abies_alba);
1314         TaxonDescription d_abies_balsamea = TaxonDescription.NewInstance(t_abies_balsamea);
1315
1316         d_abies_alba.setUuid(UUID.fromString(D_ABIES_ALBA_UUID));
1317         d_abies_balsamea.setUuid(UUID.fromString(D_ABIES_BALSAMEA_UUID));
1318
1319
1320         // CommonTaxonName
1321         d_abies_alba.addElement(CommonTaxonName.NewInstance("Wei"+UTF8.SHARP_S+"tanne", Language.GERMAN()));
1322         d_abies_alba.addElement(CommonTaxonName.NewInstance("silver fir", Language.ENGLISH()));
1323         d_abies_alba.addElement(Distribution
1324                 .NewInstance(
1325                         germany,
1326                         PresenceTerm.NATIVE()));
1327         d_abies_alba.addElement(Distribution
1328                 .NewInstance(
1329                         russia,
1330                         AbsenceTerm.ABSENT()));
1331
1332         // TextData
1333         d_abies_balsamea
1334             .addElement(TextData
1335                     .NewInstance(
1336                             "Die Balsam-Tanne (Abies balsamea) ist eine Pflanzenart aus der Gattung der Tannen (Abies). Sie wächst im nordöstlichen Nordamerika, wo sie sowohl Tief- als auch Bergland besiedelt. Sie gilt als relativ anspruchslos gegenüber dem Standort und ist frosthart. In vielen Teilen des natürlichen Verbreitungsgebietes stellt sie die Klimaxbaumart dar.",
1337                             Language.GERMAN(), null));
1338         d_abies_balsamea
1339         .addElement(CommonTaxonName
1340                 .NewInstance(
1341                         "Balsam-Tanne",
1342                         Language.GERMAN(), null));
1343
1344         d_abies_balsamea
1345         .addElement(Distribution
1346                 .NewInstance(
1347                         canada,
1348                         PresenceTerm.PRESENT()));
1349
1350         d_abies_balsamea
1351         .addElement(Distribution
1352                 .NewInstance(
1353                         germany,
1354                         PresenceTerm.NATIVE()));
1355
1356         d_abies_balsamea
1357                 .addElement(TextData
1358                         .NewInstance(
1359                                 "Бальзам ньыв (лат. Abies balsamea) – быдмассэзлӧн пожум котырись ньыв увтырын торья вид. Ньывпуыс быдмӧ 14–20 метра вылына да овлӧ 10–60 см кыза диаметрын. Ньывпу пантасьӧ Ойвыв Америкаын.",
1360                                 Language.RUSSIAN(), null));
1361         d_abies_balsamea
1362         .addElement(CommonTaxonName
1363                 .NewInstance(
1364                         "Бальзам ньыв",
1365                         Language.RUSSIAN(), null));
1366         descriptionService.saveOrUpdate(d_abies_balsamea);
1367
1368         setComplete();
1369         endTransaction();
1370
1371
1372         writeDbUnitDataSetFile(new String[] {
1373             "TAXONBASE", "TAXONNAMEBASE",
1374             "SYNONYMRELATIONSHIP", "TAXONRELATIONSHIP",
1375             "REFERENCE", "DESCRIPTIONELEMENTBASE", "DESCRIPTIONBASE",
1376             "AGENTBASE", "HOMOTYPICALGROUP",
1377             "CLASSIFICATION", "CLASSIFICATION_TAXONNODE","TAXONNODE",
1378             "LANGUAGESTRING", "DESCRIPTIONELEMENTBASE_LANGUAGESTRING",
1379             "HIBERNATE_SEQUENCES" // IMPORTANT!!!
1380             });
1381
1382     }
1383
1384     /**
1385      *
1386      */
1387     private void refreshLuceneIndex() {
1388
1389 //        commitAndStartNewTransaction(null);
1390         commit();
1391         endTransaction();
1392         indexer.purge(DefaultProgressMonitor.NewInstance());
1393         indexer.reindex(typesToIndex, DefaultProgressMonitor.NewInstance());
1394         startNewTransaction();
1395 //        commitAndStartNewTransaction(null);
1396     }
1397
1398     /**
1399      * @param numberOfNew
1400      *
1401      */
1402     private void createRandomTaxonWithCommonName(int numberOfNew) {
1403
1404         logger.debug(String.format("creating %1$s random taxan with CommonName", numberOfNew));
1405
1406         commitAndStartNewTransaction(null);
1407
1408         Reference sec = ReferenceFactory.newBook();
1409         referenceService.save(sec);
1410
1411         for (int i = numberOfNew; i < numberOfNew; i++) {
1412             RandomStringUtils.randomAlphabetic(10);
1413             String radomName = RandomStringUtils.randomAlphabetic(5) + " " + RandomStringUtils.randomAlphabetic(10);
1414             String radomCommonName = RandomStringUtils.randomAlphabetic(10);
1415
1416             BotanicalName name = BotanicalName.NewInstance(Rank.SPECIES());
1417             name.setNameCache(radomName, true);
1418             Taxon taxon = Taxon.NewInstance(name, sec);
1419             taxonService.save(taxon);
1420
1421             TaxonDescription description = TaxonDescription.NewInstance(taxon);
1422             description.addElement(CommonTaxonName.NewInstance(radomCommonName, Language.GERMAN()));
1423             descriptionService.save(description);
1424         }
1425
1426         commitAndStartNewTransaction(null);
1427     }
1428
1429     private <T extends CdmBase> void logSearchResults(Pager<SearchResult<T>> pager, Level level, String[] docFields){
1430         if(level == null){
1431             level = Level.DEBUG;
1432         }
1433         if(logger.isEnabledFor(level)){
1434             StringBuilder b = new StringBuilder();
1435             b.append("\n");
1436             int i = 0;
1437             for(SearchResult sr : pager.getRecords()){
1438
1439                 b.append(" ").append(i++).append(" - ");
1440                 b.append("score:").append(sr.getScore()).append(", ");
1441
1442                 if(docFields != null){
1443                     b.append("docs : ");
1444                     for(Document doc : sr.getDocs()) {
1445                         b.append("<");
1446                         for(String f : docFields){
1447                             b.append(f).append(":").append(Arrays.toString(doc.getValues(f)));
1448                         }
1449                         b.append(">");
1450                     }
1451                 }
1452
1453                 CdmBase entity = sr.getEntity();
1454                 if(entity == null){
1455                     b.append("NULL");
1456                 } else {
1457                     b.append(entity.getClass().getSimpleName()).
1458                         append(" [").append(entity.getId()).
1459                         append(" | ").append(entity.getUuid()).append("] : ").
1460                         append(entity.toString());
1461
1462                 }
1463                 b.append("\n");
1464             }
1465             logger.log(level, b);
1466         }
1467     }
1468
1469 }