3 * Copyright (C) 2011 EDIT
4 * European Distributed Institute of Taxonomy
5 * http://www.e-taxonomy.eu
7 * The contents of this file are subject to the Mozilla Public License Version 1.1
8 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.api
.service
.search
;
12 import java
.io
.IOException
;
13 import java
.lang
.reflect
.Field
;
14 import java
.util
.ArrayList
;
15 import java
.util
.Iterator
;
16 import java
.util
.List
;
18 import org
.apache
.log4j
.Logger
;
19 import org
.apache
.lucene
.index
.IndexReader
;
20 import org
.apache
.lucene
.index
.IndexWriterConfig
;
21 import org
.apache
.lucene
.search
.spell
.Dictionary
;
22 import org
.apache
.lucene
.search
.spell
.LuceneDictionary
;
23 import org
.apache
.lucene
.search
.spell
.SpellChecker
;
24 import org
.apache
.lucene
.store
.Directory
;
25 import org
.hibernate
.CacheMode
;
26 import org
.hibernate
.FlushMode
;
27 import org
.hibernate
.ScrollMode
;
28 import org
.hibernate
.ScrollableResults
;
29 import org
.hibernate
.Session
;
30 import org
.hibernate
.Transaction
;
31 import org
.hibernate
.search
.FullTextSession
;
32 import org
.hibernate
.search
.Search
;
33 import org
.hibernate
.search
.engine
.spi
.SearchFactoryImplementor
;
34 import org
.hibernate
.search
.indexes
.impl
.DirectoryBasedIndexManager
;
35 import org
.hibernate
.search
.indexes
.spi
.IndexManager
;
36 import org
.springframework
.beans
.factory
.annotation
.Autowired
;
37 import org
.springframework
.orm
.hibernate4
.HibernateTransactionManager
;
38 import org
.springframework
.stereotype
.Component
;
39 import org
.springframework
.transaction
.PlatformTransactionManager
;
40 import org
.springframework
.transaction
.annotation
.Transactional
;
42 import eu
.etaxonomy
.cdm
.common
.monitor
.IProgressMonitor
;
43 import eu
.etaxonomy
.cdm
.common
.monitor
.NullProgressMonitor
;
44 import eu
.etaxonomy
.cdm
.common
.monitor
.RestServiceProgressMonitor
;
45 import eu
.etaxonomy
.cdm
.common
.monitor
.SubProgressMonitor
;
46 import eu
.etaxonomy
.cdm
.config
.Configuration
;
47 import eu
.etaxonomy
.cdm
.model
.common
.CdmBase
;
48 import eu
.etaxonomy
.cdm
.model
.description
.DescriptionElementBase
;
49 import eu
.etaxonomy
.cdm
.model
.name
.NonViralName
;
50 import eu
.etaxonomy
.cdm
.model
.name
.TaxonNameBase
;
51 import eu
.etaxonomy
.cdm
.model
.occurrence
.SpecimenOrObservationBase
;
52 import eu
.etaxonomy
.cdm
.model
.taxon
.Classification
;
53 import eu
.etaxonomy
.cdm
.model
.taxon
.TaxonBase
;
56 * @author Andreas Kohlbecker
62 public class CdmMassIndexer
implements ICdmMassIndexer
{
64 public static final Logger logger
= Logger
.getLogger(CdmMassIndexer
.class);
66 private static final int BATCH_SIZE
= 2000;
68 public HibernateTransactionManager transactionManager
;
71 public void setTransactionManager(PlatformTransactionManager transactionManager
) {
72 this.transactionManager
= (HibernateTransactionManager
)transactionManager
;
75 protected Session
getSession(){
76 Session session
= transactionManager
.getSessionFactory().getCurrentSession();
80 protected <T
extends CdmBase
>void reindex(Class
<T
> type
, IProgressMonitor monitor
) {
82 FullTextSession fullTextSession
= Search
.getFullTextSession(getSession());
84 fullTextSession
.setFlushMode(FlushMode
.MANUAL
);
85 fullTextSession
.setCacheMode(CacheMode
.IGNORE
);
87 logger
.info("start indexing " + type
.getName());
88 monitor
.subTask("indexing " + type
.getSimpleName());
90 Long countResult
= countEntities(type
);
91 int numOfBatches
= calculateNumOfBatches(countResult
);
93 SubProgressMonitor subMonitor
= new SubProgressMonitor(monitor
, 1);
94 subMonitor
.beginTask("Indexing " + type
.getSimpleName(), numOfBatches
);
96 // Scrollable results will avoid loading too many objects in memory
97 ScrollableResults results
= fullTextSession
.createCriteria(type
).setFetchSize(BATCH_SIZE
).scroll(ScrollMode
.FORWARD_ONLY
);
99 int batchesWorked
= 0;
102 while (results
.next()) {
104 fullTextSession
.index(results
.get(0)); // index each element
105 if (index
% BATCH_SIZE
== 0 || index
== countResult
) {
107 fullTextSession
.flushToIndexes(); // apply changes to indexes
108 fullTextSession
.clear(); // clear since the queue is processed
109 subMonitor
.worked(1);
110 logger
.info("\tbatch " + batchesWorked
+ "/" + numOfBatches
+ " processed");
111 //if(index / BATCH_SIZE > 10 ) break;
114 } catch (RuntimeException e
) {
115 //TODO better means to notify that the process has been stopped, using the STOPPED_WORK_INDICATOR is only a hack
116 monitor
.worked(RestServiceProgressMonitor
.STOPPED_WORK_INDICATOR
);
120 logger
.info("end indexing " + type
.getName());
130 protected <T
extends CdmBase
> void createDictionary(Class
<T
> type
, IProgressMonitor monitor
) {
131 String indexName
= null;
132 if(type
.isAnnotationPresent(org
.hibernate
.search
.annotations
.Indexed
.class)) {
133 indexName
= type
.getAnnotation(org
.hibernate
.search
.annotations
.Indexed
.class).index();
135 //TODO:give some indication that this class is infact not indexed
138 SearchFactoryImplementor searchFactory
= (SearchFactoryImplementor
)Search
.getFullTextSession(getSession()).getSearchFactory();
139 IndexManager indexManager
= searchFactory
.getAllIndexesManager().getIndexManager(indexName
);
140 IndexReader indexReader
= searchFactory
.getIndexReaderAccessor().open(type
);
141 List
<String
> idFields
= getIndexedDeclaredFields(type
);
143 monitor
.subTask("creating dictionary " + type
.getSimpleName());
145 SubProgressMonitor subMonitor
= new SubProgressMonitor(monitor
, 1);
146 subMonitor
.beginTask("Creating dictionary " + type
.getSimpleName(), 1);
148 Directory directory
= ((DirectoryBasedIndexManager
) indexManager
).getDirectoryProvider().getDirectory();
149 SpellChecker spellChecker
= null;
151 spellChecker
= new SpellChecker(directory
);
152 Iterator
<String
> itr
= idFields
.iterator();
153 while(itr
.hasNext()) {
154 String indexedField
= itr
.next();
155 logger
.info("creating dictionary for field " + indexedField
);
156 Dictionary dictionary
= new LuceneDictionary(indexReader
, indexedField
);
157 IndexWriterConfig iwc
= new IndexWriterConfig(Configuration
.luceneVersion
, searchFactory
.getAnalyzer(type
));
158 spellChecker
.indexDictionary(dictionary
, iwc
, true);
160 subMonitor
.internalWorked(1);
161 } catch (IOException e
) {
162 logger
.error("IOException when creating dictionary", e
);
163 //TODO better means to notify that the process has been stopped, using the STOPPED_WORK_INDICATOR is only a hack
164 monitor
.worked(RestServiceProgressMonitor
.STOPPED_WORK_INDICATOR
);
166 } catch (RuntimeException e
) {
167 logger
.error("RuntimeException when creating dictionary", e
);
168 //TODO better means to notify that the process has been stopped, using the STOPPED_WORK_INDICATOR is only a hack
169 monitor
.worked(RestServiceProgressMonitor
.STOPPED_WORK_INDICATOR
);
172 searchFactory
.getIndexReaderAccessor().close(indexReader
);
174 if (spellChecker
!= null) {
176 logger
.info("closing spellchecker ");
177 spellChecker
.close();
178 } catch (IOException e
) {
179 logger
.error("IOException when closing spellchecker", e
);
183 logger
.info("end creating dictionary " + type
.getName());
191 private int calculateNumOfBatches(Long countResult
) {
192 Long numOfBatches
= countResult
> 0 ?
((countResult
-1)/BATCH_SIZE
)+1 : 0;
193 return numOfBatches
.intValue();
200 private <T
> Long
countEntities(Class
<T
> type
) {
201 Object countResultObj
= getSession().createQuery("select count(*) from " + type
.getName()).uniqueResult();
202 Long countResult
= (Long
)countResultObj
;
206 protected <T
extends CdmBase
>void purge(Class
<T
> type
, IProgressMonitor monitor
) {
208 FullTextSession fullTextSession
= Search
.getFullTextSession(getSession());
209 logger
.info("purging " + type
.getName());
210 fullTextSession
.purgeAll(type
);
213 SearchFactoryImplementor searchFactory
= (SearchFactoryImplementor
)Search
.getFullTextSession(getSession()).getSearchFactory();
214 IndexManager indexManager
= searchFactory
.getAllIndexesManager().getIndexManager(type
.getName());
215 Directory directory
= ((DirectoryBasedIndexManager
) indexManager
).getDirectoryProvider().getDirectory();
216 SpellChecker spellChecker
= null;
218 spellChecker
= new SpellChecker(directory
);
219 spellChecker
.clearIndex();
220 } catch (IOException e
) {
221 logger
.error("IOException when creating dictionary", e
);
222 //TODO better means to notify that the process has been stopped, using the STOPPED_WORK_INDICATOR is only a hack
223 //monitor.worked(RestServiceProgressMonitor.STOPPED_WORK_INDICATOR);
227 if (spellChecker
!= null) {
229 logger
.info("closing spellchecker ");
230 spellChecker
.close();
231 } catch (IOException e
) {
232 logger
.error("IOException when closing spellchecker", e
);
239 * @see eu.etaxonomy.cdm.database.IMassIndexer#reindex()
242 public void reindex(IProgressMonitor monitor
){
245 monitor
= new NullProgressMonitor();
248 monitor
.setTaskName("CdmMassIndexer");
249 int steps
= indexedClasses().length
+ 1; // +1 for optimize
250 monitor
.beginTask("Reindexing " + indexedClasses().length
+ " classes", steps
);
252 for(Class
<?
extends CdmBase
> type
: indexedClasses()){
253 reindex(type
, monitor
);
254 // clear the session after each class to free memory
255 getSession().clear();
265 public void createDictionary(IProgressMonitor monitor
) {
267 monitor
= new NullProgressMonitor();
270 monitor
.setTaskName("CdmMassIndexer_Dictionary");
271 int steps
= dictionaryClasses().length
; // +1 for optimize
272 monitor
.beginTask("Creating Dictionary " + dictionaryClasses().length
+ " classes", steps
);
274 for(Class type
: dictionaryClasses()){
275 createDictionary(type
, monitor
);
281 protected void optimize() {
283 FullTextSession fullTextSession
= Search
.getFullTextSession(getSession());
284 fullTextSession
.getSearchFactory().optimize();
285 fullTextSession
.flushToIndexes();
286 fullTextSession
.clear();
292 private int totalBatchCount() {
293 int totalNumOfBatches
= 0;
294 for(Class type
: indexedClasses()){
295 totalNumOfBatches
+= calculateNumOfBatches(countEntities(type
));
297 return totalNumOfBatches
;
301 * @see eu.etaxonomy.cdm.database.IMassIndexer#purge()
304 public void purge(IProgressMonitor monitor
){
307 monitor
= new NullProgressMonitor();
310 monitor
.setTaskName("CdmMassIndexer");
311 int steps
= indexedClasses().length
+ 1; // +1 for optimize
312 monitor
.beginTask("Purging " + indexedClasses().length
+ " classes", steps
);
314 for(Class
<?
extends CdmBase
> type
: indexedClasses()){
315 purge(type
, monitor
);
319 // // need to commit and start new transaction before optimizing
320 FullTextSession fullTextSession
= Search
.getFullTextSession(getSession());
321 Transaction tx
= fullTextSession
.getTransaction();
323 fullTextSession
.beginTransaction(); // will be committed automatically at the end of this method since this class is transactional
333 * Returns a list of declared indexable fields within a class through reflection.
338 private List
<String
> getIndexedDeclaredFields(Class clazz
) {
339 List
<String
> idFields
= new ArrayList
<String
>();
340 if(clazz
.isAnnotationPresent(org
.hibernate
.search
.annotations
.Indexed
.class)) {
341 Field
[] declaredFields
= clazz
.getDeclaredFields();
342 for(int i
=0;i
<declaredFields
.length
;i
++ ) {
343 logger
.info("checking field " + declaredFields
[i
].getName());
344 if(declaredFields
[i
].isAnnotationPresent(org
.hibernate
.search
.annotations
.Field
.class) ||
345 declaredFields
[i
].isAnnotationPresent(org
.hibernate
.search
.annotations
.Fields
.class)) {
346 idFields
.add(declaredFields
[i
].getName());
347 logger
.info("adding field " + declaredFields
[i
].getName());
356 @SuppressWarnings("unchecked")
358 public Class
<?
extends CdmBase
>[] indexedClasses() {
360 DescriptionElementBase
.class,
361 Classification
.class,
364 SpecimenOrObservationBase
.class
372 public Class
[] dictionaryClasses() {