cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IdentifiableServiceBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import java.util.ArrayList;
  13 import java.util.Arrays;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22 import org.hibernate.criterion.Criterion;
  23 import org.springframework.transaction.annotation.Transactional;
  24
  25 import eu.etaxonomy.cdm.api.service.config.IIdentifiableEntityServiceConfigurator;
  26 import eu.etaxonomy.cdm.api.service.dto.IdentifiedEntityDTO;
  27 import eu.etaxonomy.cdm.api.service.dto.MarkedEntityDTO;
  28 import eu.etaxonomy.cdm.api.service.pager.Pager;
  29 import eu.etaxonomy.cdm.api.service.pager.impl.DefaultPagerImpl;
  30 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  31 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  32 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  33 import eu.etaxonomy.cdm.model.common.CdmBase;
  34 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  35 import eu.etaxonomy.cdm.model.common.ISourceable;
  36 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LSID;
  39 import eu.etaxonomy.cdm.model.common.MarkerType;
  40 import eu.etaxonomy.cdm.model.media.Rights;
  41 import eu.etaxonomy.cdm.model.name.INonViralName;
  42 import eu.etaxonomy.cdm.model.name.ITaxonName;
  43 import eu.etaxonomy.cdm.model.name.TaxonName;
  44 import eu.etaxonomy.cdm.model.reference.Reference;
  45 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  46 import eu.etaxonomy.cdm.persistence.dao.common.IIdentifiableDao;
  47 import eu.etaxonomy.cdm.persistence.dao.hibernate.HibernateBeanInitializer;
  48 import eu.etaxonomy.cdm.persistence.dao.initializer.AutoPropertyInitializer;
  49 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  50 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  51 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  52 import eu.etaxonomy.cdm.persistence.query.OrderHint.SortOrder;
  53 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
  54 import eu.etaxonomy.cdm.strategy.match.DefaultMatchStrategy;
  55 import eu.etaxonomy.cdm.strategy.match.IMatchStrategy;
  56 import eu.etaxonomy.cdm.strategy.match.IMatchable;
  57 import eu.etaxonomy.cdm.strategy.match.MatchException;
  58 import eu.etaxonomy.cdm.strategy.merge.IMergable;
  59 import eu.etaxonomy.cdm.strategy.merge.IMergeStrategy;
  60 import eu.etaxonomy.cdm.strategy.merge.MergeException;
  61
  62 public abstract class IdentifiableServiceBase<T extends IdentifiableEntity, DAO extends IIdentifiableDao<T>> extends AnnotatableServiceBase<T,DAO>
  63                                                 implements IIdentifiableEntityService<T>{
  64
  65
  66         protected static final int UPDATE_TITLE_CACHE_DEFAULT_STEP_SIZE = 1000;
  67         protected static final  Logger logger = Logger.getLogger(IdentifiableServiceBase.class);
  68
  69         @Override
  70         @Transactional(readOnly = true)
  71         public Pager<Rights> getRights(T t, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
  72         Integer numberOfResults = dao.countRights(t);
  73
  74                 List<Rights> results = new ArrayList<Rights>();
  75                 if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
  76                         results = dao.getRights(t, pageSize, pageNumber,propertyPaths);
  77                 }
  78
  79                 return new DefaultPagerImpl<Rights>(pageNumber, numberOfResults, pageSize, results);
  80         }
  81
  82         @Override
  83         @Transactional(readOnly = true)
  84         public Pager<IdentifiableSource> getSources(T t, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
  85                  Integer numberOfResults = dao.countSources(t);
  86
  87                  List<IdentifiableSource> results = new ArrayList<IdentifiableSource>();
  88                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
  89                          results = dao.getSources(t, pageSize, pageNumber,propertyPaths);
  90                  }
  91
  92                  return new DefaultPagerImpl<IdentifiableSource>(pageNumber, numberOfResults, pageSize, results);
  93         }
  94
  95
  96         @Transactional(readOnly = false)
  97         @Override
  98         public T replace(T x, T y) {
  99                 return dao.replace(x, y);
 100         }
 101         /**
 102          * FIXME Candidate for harmonization
 103          * Given that this method is strongly typed, and generic, could we not simply expose it as
 104          * List<T> findByTitle(String title) as it is somewhat less cumbersome. Admittedly, I don't
 105          * understand what is going on with the configurators etc. so maybe there is a good reason for
 106          * the design of this method.
 107          * @param title
 108          * @return
 109          */
 110         @Transactional(readOnly = true)
 111         protected List<T> findCdmObjectsByTitle(String title){
 112                 return ((IIdentifiableDao)dao).findByTitle(title);
 113         }
 114
 115         @Transactional(readOnly = true)
 116         protected List<T> findCdmObjectsByTitle(String title, Class<T> clazz){
 117                 return ((IIdentifiableDao)dao).findByTitleAndClass(title, clazz);
 118         }
 119         @Transactional(readOnly = true)
 120         protected List<T> findCdmObjectsByTitle(String title, CdmBase sessionObject){
 121                 return ((IIdentifiableDao)dao).findByTitle(title, sessionObject);
 122         }
 123
 124         /*
 125          * TODO - Migrated from CommonServiceBase
 126          *  (non-Javadoc)
 127          * @see eu.etaxonomy.cdm.api.service.ICommonService#getSourcedObjectById(java.lang.String, java.lang.String)
 128          */
 129         @Transactional(readOnly = true)
 130         @Override
 131         public ISourceable getSourcedObjectByIdInSource(Class clazz, String idInSource, String idNamespace) {
 132                 ISourceable result = null;
 133
 134                 List<T> list = dao.findOriginalSourceByIdInSource(idInSource, idNamespace);
 135                 if (! list.isEmpty()){
 136                         result = list.get(0);
 137                 }
 138                 return result;
 139         }
 140
 141         @Transactional(readOnly = true)
 142         @Override
 143         public List<UuidAndTitleCache<T>> getUuidAndTitleCache(Integer limit, String pattern) {
 144                 return dao.getUuidAndTitleCache(limit, pattern);
 145         }
 146
 147     @Transactional(readOnly = true)
 148     @Override
 149     public <S extends T> List<UuidAndTitleCache<S>> getUuidAndTitleCache(Class<S> clazz,Integer limit, String pattern) {
 150         return dao.getUuidAndTitleCache(clazz, limit, pattern);
 151     }
 152
 153     @Transactional(readOnly = true)
 154     @Override
 155     public String getTitleCache(UUID uuid, boolean refresh){
 156         return dao.getTitleCache(uuid, refresh);
 157     }
 158
 159         @Transactional(readOnly = true)
 160         @Override
 161         public Pager<T> findByTitle(Class<? extends T> clazz, String queryString,MatchMode matchmode, List<Criterion> criteria, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 162                  long numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria);
 163
 164                  List<T> results = new ArrayList<T>();
 165                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 166                                 results = dao.findByTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths);
 167                  }
 168
 169                   return new DefaultPagerImpl<T>(pageNumber, numberOfResults, pageSize, results);
 170         }
 171
 172         @Transactional(readOnly = true)
 173         @Override
 174         public Pager<T> findByTitle(IIdentifiableEntityServiceConfigurator<T> config){
 175                 return findByTitle(config.getClazz(), config.getTitleSearchStringSqlized(), config.getMatchMode(), config.getCriteria(), config.getPageSize(), config.getPageNumber(), config.getOrderHints(), config.getPropertyPaths());
 176         }
 177
 178         @Transactional(readOnly = true)
 179         @Override
 180         public List<T> listByTitle(Class<? extends T> clazz, String queryString,MatchMode matchmode, List<Criterion> criteria, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 181                  long numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria);
 182
 183                  List<T> results = new ArrayList<T>();
 184                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 185                                 results = dao.findByTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths);
 186                  }
 187                  return results;
 188         }
 189
 190         @Transactional(readOnly = true)
 191         @Override
 192         public Pager<T> findTitleCache(Class<? extends T> clazz, String queryString, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, MatchMode matchMode){
 193                 long numberOfResults = dao.countTitleCache(clazz, queryString, matchMode);
 194
 195                  List<T> results = new ArrayList<T>();
 196                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 197                                 results = dao.findTitleCache(clazz, queryString, pageSize, pageNumber, orderHints, matchMode);
 198                  }
 199                  long r = 0;
 200                  r += numberOfResults;
 201
 202                   return new DefaultPagerImpl<T>(pageNumber, r , pageSize, results);
 203         }
 204
 205         @Transactional(readOnly = true)
 206         @Override
 207         public List<T> listByReferenceTitle(Class<? extends T> clazz, String queryString,MatchMode matchmode, List<Criterion> criteria, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 208                  long numberOfResults = dao.countByReferenceTitle(clazz, queryString, matchmode, criteria);
 209
 210                  List<T> results = new ArrayList<T>();
 211                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 212                                 results = dao.findByReferenceTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths);
 213                  }
 214                  return results;
 215         }
 216
 217         @Transactional(readOnly = true)
 218         @Override
 219         public T find(LSID lsid) {
 220                 return dao.find(lsid);
 221         }
 222
 223         @Transactional(readOnly = true)
 224         @Override
 225         public Pager<T> search(Class<? extends T> clazz, String queryString, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 226         Integer numberOfResults = dao.count(clazz,queryString);
 227
 228                 List<T> results = new ArrayList<T>();
 229                 if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 230                         results = dao.search(clazz,queryString, pageSize, pageNumber, orderHints, propertyPaths);
 231                 }
 232
 233                 return new DefaultPagerImpl<T>(pageNumber, numberOfResults, pageSize, results);
 234         }
 235
 236         @Override
 237         @Transactional(readOnly = false)
 238         public void updateTitleCache() {
 239                 updateTitleCache(null, null, null, null);
 240         }
 241
 242         @Transactional(readOnly = false)  //TODO check transactional behaviour, e.g. what happens with the session if count is very large
 243         protected <S extends T > void updateTitleCacheImpl(Class<S> clazz, Integer stepSize, IIdentifiableEntityCacheStrategy<T> cacheStrategy, IProgressMonitor monitor) {
 244                 if (stepSize == null){
 245                         stepSize = UPDATE_TITLE_CACHE_DEFAULT_STEP_SIZE;
 246                 }
 247                 if (monitor == null){
 248                         monitor = DefaultProgressMonitor.NewInstance();
 249                 }
 250
 251                 int count = dao.count(clazz);
 252                 monitor.beginTask("update titles", count);
 253                 int worked = 0;
 254                 for(int i = 0 ; i < count ; i = i + stepSize){
 255                         // not sure if such strict ordering is necessary here, but for safety reasons I do it
 256                         ArrayList<OrderHint> orderHints = new ArrayList<OrderHint>();
 257                         orderHints.add( new OrderHint("id", OrderHint.SortOrder.ASCENDING));
 258
 259
 260                         Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> oldAutoInit = switchOfAutoinitializer();
 261                         List<S> list = this.list(clazz, stepSize, i, orderHints, null);
 262                         switchOnOldAutoInitializer(oldAutoInit);
 263
 264                         List<T> entitiesToUpdate = new ArrayList<T>();
 265                         for (T entity : list){
 266                                 HibernateProxyHelper.deproxy(entity, clazz);
 267                                 if (entity.isProtectedTitleCache() == false){
 268                                         updateTitleCacheForSingleEntity(cacheStrategy, entitiesToUpdate, entity);
 269                                 }
 270                                 worked++;
 271                         }
 272                         for (T entity: entitiesToUpdate){
 273                                 if (entity.getTitleCache() != null){
 274                                         //System.err.println(entity.getTitleCache());
 275                                 }else{
 276                                     //System.err.println("no titleCache" + ((TaxonName)entity).getNameCache());
 277                                 }
 278                         }
 279                         saveOrUpdate(entitiesToUpdate);
 280                         monitor.worked(list.size());
 281                         if (monitor.isCanceled()){
 282                                 monitor.done();
 283                                 return;
 284                         }
 285                 }
 286                 monitor.done();
 287         }
 288
 289         /**
 290          * Brings back all auto initializers to the bean initializer
 291          * @see #switchOfAutoinitializer()
 292          * @param oldAutoInit
 293          */
 294         protected void switchOnOldAutoInitializer(
 295                         Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> oldAutoInit) {
 296                 HibernateBeanInitializer initializer = (HibernateBeanInitializer)this.appContext.getBean("defaultBeanInitializer");
 297                 initializer.setBeanAutoInitializers(oldAutoInit);
 298         }
 299
 300         /**
 301          * Removes all auto initializers from the bean initializer
 302          *
 303          * @see #switchOnOldAutoInitializer(Map)
 304          * @return
 305          */
 306         protected Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> switchOfAutoinitializer() {
 307                 HibernateBeanInitializer initializer = (HibernateBeanInitializer)this.appContext.getBean("defaultBeanInitializer");
 308                 Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> oldAutoInitializers = initializer.getBeanAutoInitializers();
 309                 Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> map = new HashMap<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>>();
 310                 initializer.setBeanAutoInitializers(map);
 311                 return oldAutoInitializers;
 312         }
 313
 314         /**
 315          * @param cacheStrategy
 316          * @param entitiesToUpdate
 317          * @param entity
 318          */
 319         /**
 320          * @param cacheStrategy
 321          * @param entitiesToUpdate
 322          * @param entity
 323          */
 324         @SuppressWarnings("unchecked")
 325         private void updateTitleCacheForSingleEntity(
 326                         IIdentifiableEntityCacheStrategy<T> cacheStrategy,
 327                         List<T> entitiesToUpdate,
 328                         T entity) {
 329
 330                 //assert (entity.isProtectedTitleCache() == false );
 331             entity = CdmBase.deproxy(entity);
 332
 333                 //exclude recursive inreferences
 334                 if (entity.isInstanceOf(Reference.class)){
 335                         Reference ref = (Reference)entity;
 336                         if (ref.getInReference() != null && ref.getInReference().equals(ref)){
 337                                 return;
 338                         }
 339                 }
 340
 341
 342                 //define the correct cache strategy
 343                 IIdentifiableEntityCacheStrategy entityCacheStrategy = cacheStrategy;
 344                 if (entityCacheStrategy == null){
 345                         entityCacheStrategy = entity.getCacheStrategy();
 346                         //FIXME find out why the wrong cache strategy is loaded here, see #1876
 347                         if (entity instanceof Reference){
 348                                 entityCacheStrategy = ReferenceFactory.newReference(((Reference)entity).getType()).getCacheStrategy();
 349                         }
 350                 }
 351
 352
 353                 //old titleCache
 354                 entity.setProtectedTitleCache(true);
 355
 356                 String oldTitleCache = entity.getTitleCache();
 357                 entity.setTitleCache(oldTitleCache, false);   //before we had entity.setProtectedTitleCache(false) but this deleted the titleCache itself
 358                 entity.setCacheStrategy(entityCacheStrategy);
 359                 //NonViralNames and Reference have more caches //TODO handle in NameService
 360                 String oldNameCache = null;
 361                 String oldFullTitleCache = null;
 362                 String oldAbbrevTitleCache = null;
 363                 if (entity instanceof TaxonName ){
 364                     if (((TaxonName) entity).isNonViral()) {
 365                 try{
 366                         INonViralName nvn = (INonViralName) entity;
 367                         if (!nvn.isProtectedNameCache()){
 368                             nvn.setProtectedNameCache(true);
 369                                 oldNameCache = nvn.getNameCache();
 370                                 nvn.setProtectedNameCache(false);
 371                         }
 372                         if (!nvn.isProtectedFullTitleCache()){
 373                             nvn.setProtectedFullTitleCache(true);
 374                                 oldFullTitleCache = nvn.getFullTitleCache();
 375                                 nvn.setProtectedFullTitleCache(false);
 376                         }
 377                 }catch(ClassCastException e){
 378                         System.out.println("entity: " + entity.getTitleCache());
 379                 }
 380             }
 381
 382                 }else if (entity instanceof Reference){
 383                         Reference ref = (Reference) entity;
 384                         if (!ref.isProtectedAbbrevTitleCache()){
 385                                 ref.setProtectedAbbrevTitleCache(true);
 386                                 oldAbbrevTitleCache = ref.getAbbrevTitleCache();
 387                                 ref.setProtectedAbbrevTitleCache(false);
 388                         }
 389                 }
 390                 setOtherCachesNull(entity);
 391                 String newTitleCache= null;
 392                 INonViralName nvn = null; //TODO find better solution
 393                 try{
 394                         if (entity instanceof TaxonName){
 395                                 nvn = (ITaxonName) entity;
 396                                 newTitleCache = entityCacheStrategy.getTitleCache(nvn);
 397                         } else{
 398                                  newTitleCache = entityCacheStrategy.getTitleCache(entity);
 399                         }
 400                 }catch (ClassCastException e){
 401                         nvn = HibernateProxyHelper.deproxy(entity, TaxonName.class);
 402                         newTitleCache = entityCacheStrategy.getTitleCache(nvn);
 403                 }
 404
 405                 if ( oldTitleCache == null   || ! oldTitleCache.equals(newTitleCache) ){
 406                         entity.setTitleCache(null, false);
 407                         String newCache = entity.getTitleCache();
 408
 409                         if (newCache == null){
 410                                 logger.warn("newCache should never be null");
 411                         }
 412                         if (oldTitleCache == null){
 413                                 logger.info("oldTitleCache should never be null");
 414                         }
 415                         if (nvn != null){
 416                                 nvn.getNameCache();
 417                                 nvn.getFullTitleCache();
 418                         }
 419                         if (entity instanceof Reference){
 420                                 Reference ref = (Reference) entity;
 421                                 ref.getAbbrevTitleCache();
 422                         }
 423                         entitiesToUpdate.add(entity);
 424                 }else if (nvn != null){
 425                         String newNameCache = nvn.getNameCache();
 426                         String newFullTitleCache = nvn.getFullTitleCache();
 427                         if ((oldNameCache == null && !nvn.isProtectedNameCache()) || (oldNameCache != null && !oldNameCache.equals(newNameCache))){
 428                                 entitiesToUpdate.add(entity);
 429                         }else if ((oldFullTitleCache == null && !nvn.isProtectedFullTitleCache()) || (oldFullTitleCache != null && !oldFullTitleCache.equals(newFullTitleCache))){
 430                                 entitiesToUpdate.add(entity);
 431                         }
 432                 }else if (entity instanceof Reference){
 433                         Reference ref = (Reference) entity;
 434                         String newAbbrevTitleCache = ref.getAbbrevTitleCache();
 435                         if ( (oldAbbrevTitleCache == null && !ref.isProtectedAbbrevTitleCache() ) || (oldAbbrevTitleCache != null && !oldAbbrevTitleCache.equals(newAbbrevTitleCache))){
 436                                 entitiesToUpdate.add(entity);
 437                         }
 438                 }
 439
 440         }
 441
 442
 443         /**
 444          * Needs override if not only the title cache should be set to null to
 445          * generate the correct new title cache
 446          */
 447         protected void setOtherCachesNull(T entity) {
 448                 return;
 449         }
 450
 451         private class DeduplicateState{
 452                 String lastTitleCache;
 453                 Integer pageSize = 50;
 454                 int nPages = 3;
 455                 int startPage = 0;
 456                 boolean isCompleted = false;
 457                 int result;
 458         }
 459
 460         @Override
 461         @Transactional(readOnly = false)
 462         public int deduplicate(Class<? extends T> clazz, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) {
 463                 DeduplicateState dedupState = new DeduplicateState();
 464
 465                 if (clazz == null){
 466                         logger.warn("Deduplication clazz must not be null!");
 467                         return 0;
 468                 }
 469                 if (! ( IMatchable.class.isAssignableFrom(clazz) && IMergable.class.isAssignableFrom(clazz) )  ){
 470                         logger.warn("Deduplication implemented only for classes implementing IMatchable and IMergeable. No deduplication performed!");
 471                         return 0;
 472                 }
 473                 Class matchableClass = clazz;
 474                 if (matchStrategy == null){
 475                         matchStrategy = DefaultMatchStrategy.NewInstance(matchableClass);
 476                 }
 477                 List<T> nextGroup = new ArrayList<T>();
 478
 479                 int result = 0;
 480 //              double countTotal = count(clazz);
 481 //
 482 //              Number countPagesN = Math.ceil(countTotal/dedupState.pageSize.doubleValue()) ;
 483 //              int countPages = countPagesN.intValue();
 484 //
 485
 486                 List<OrderHint> orderHints = Arrays.asList(new OrderHint[]{new OrderHint("titleCache", SortOrder.ASCENDING)});
 487
 488                 while (! dedupState.isCompleted){
 489                         //get x page sizes
 490                         List<T> objectList = getPages(clazz, dedupState, orderHints);
 491                         //after each page check if any changes took place
 492                         int nUnEqualPages = handleAllPages(objectList, dedupState, nextGroup, matchStrategy, mergeStrategy);
 493                         nUnEqualPages = nUnEqualPages + dedupState.pageSize * dedupState.startPage;
 494                         //refresh start page counter
 495                         int finishedPages = nUnEqualPages / dedupState.pageSize;
 496                         dedupState.startPage = finishedPages;
 497                 }
 498
 499                 result += handleLastGroup(nextGroup, matchStrategy, mergeStrategy);
 500                 return result;
 501         }
 502
 503
 504         private int handleAllPages(List<T> objectList, DeduplicateState dedupState, List<T> nextGroup, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) {
 505                 int nUnEqual = 0;
 506                 for (T object : objectList){
 507                         String currentTitleCache = object.getTitleCache();
 508                         if (currentTitleCache != null && currentTitleCache.equals(dedupState.lastTitleCache)){
 509                                 //=titleCache
 510                                 nextGroup.add(object);
 511                         }else{
 512                                 //<> titleCache
 513                                 dedupState.result += handleLastGroup(nextGroup, matchStrategy, mergeStrategy);
 514                                 nextGroup = new ArrayList<T>();
 515                                 nextGroup.add(object);
 516                                 nUnEqual++;
 517                         }
 518                         dedupState.lastTitleCache = currentTitleCache;
 519                 }
 520                 handleLastGroup(nextGroup, matchStrategy, mergeStrategy);
 521                 return nUnEqual;
 522         }
 523
 524         private List<T> getPages(Class<? extends T> clazz, DeduplicateState dedupState, List<OrderHint> orderHints) {
 525                 List<T> result = new ArrayList<T>();
 526                 for (int pageNo = dedupState.startPage; pageNo < dedupState.startPage + dedupState.nPages; pageNo++){
 527                         List<T> objectList = listByTitle(clazz, null, null, null, dedupState.pageSize, pageNo, orderHints, null);
 528                         result.addAll(objectList);
 529                 }
 530                 if (result.size()< dedupState.nPages * dedupState.pageSize ){
 531                         dedupState.isCompleted = true;
 532                 }
 533                 return result;
 534         }
 535
 536         private int handleLastGroup(List<T> group, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) {
 537                 int result = 0;
 538                 int size = group.size();
 539                 Set<Integer> exclude = new HashSet<Integer>();  //set to collect all objects, that have been merged already
 540                 for (int i = 0; i < size - 1; i++){
 541                         if (exclude.contains(i)){
 542                                 continue;
 543                         }
 544                         for (int j = i + 1; j < size; j++){
 545                                 if (exclude.contains(j)){
 546                                         continue;
 547                                 }
 548                                 T firstObject = group.get(i);
 549                                 T secondObject = group.get(j);
 550
 551                                 try {
 552                                         if (matchStrategy.invoke((IMatchable)firstObject, (IMatchable)secondObject)){
 553                                                 commonService.merge((IMergable)firstObject, (IMergable)secondObject, mergeStrategy);
 554                                                 exclude.add(j);
 555                                                 result++;
 556                                         }
 557                                 } catch (MatchException e) {
 558                                         logger.warn("MatchException when trying to match " + firstObject.getTitleCache());
 559                                         e.printStackTrace();
 560                                 } catch (MergeException e) {
 561                                         logger.warn("MergeException when trying to merge " + firstObject.getTitleCache());
 562                                         e.printStackTrace();
 563                                 }
 564                         }
 565                 }
 566                 return result;
 567         }
 568
 569         @Transactional(readOnly = true)
 570         @Override
 571         public Integer countByTitle(Class<? extends T> clazz, String queryString,MatchMode matchmode, List<Criterion> criteria){
 572                  long numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria);
 573
 574                  return ((Number)numberOfResults).intValue();
 575         }
 576
 577         @Transactional(readOnly = true)
 578         @Override
 579         public Integer countByTitle(IIdentifiableEntityServiceConfigurator<T> config){
 580                 return countByTitle(config.getClazz(), config.getTitleSearchStringSqlized(),
 581                                 config.getMatchMode(), config.getCriteria());
 582
 583         }
 584
 585         @Override
 586         @Transactional(readOnly = true)
 587         public <S extends T> Pager<IdentifiedEntityDTO<S>> findByIdentifier(
 588                         Class<S> clazz, String identifier, DefinedTerm identifierType, MatchMode matchmode,
 589                         boolean includeEntity, Integer pageSize,
 590                         Integer pageNumber,     List<String> propertyPaths) {
 591
 592                 Integer numberOfResults = dao.countByIdentifier(clazz, identifier, identifierType, matchmode);
 593         List<Object[]> daoResults = new ArrayList<Object[]>();
 594         if(numberOfResults > 0) { // no point checking again
 595                 daoResults = dao.findByIdentifier(clazz, identifier, identifierType,
 596                                 matchmode, includeEntity, pageSize, pageNumber, propertyPaths);
 597         }
 598
 599         List<IdentifiedEntityDTO<S>> result = new ArrayList<IdentifiedEntityDTO<S>>();
 600         for (Object[] daoObj : daoResults){
 601                 if (includeEntity){
 602                         result.add(new IdentifiedEntityDTO<S>((DefinedTerm)daoObj[0], (String)daoObj[1], (S)daoObj[2]));
 603                 }else{
 604                         result.add(new IdentifiedEntityDTO<S>((DefinedTerm)daoObj[0], (String)daoObj[1], (UUID)daoObj[2], (String)daoObj[3]));
 605                 }
 606         }
 607                 return new DefaultPagerImpl<IdentifiedEntityDTO<S>>(pageNumber, numberOfResults, pageSize, result);
 608         }
 609
 610     @Override
 611     @Transactional(readOnly = true)
 612     public <S extends T> Pager<MarkedEntityDTO<S>> findByMarker(
 613             Class<S> clazz, MarkerType markerType, Boolean markerValue,
 614             boolean includeEntity, Integer pageSize,
 615             Integer pageNumber, List<String> propertyPaths) {
 616
 617         Long numberOfResults = dao.countByMarker(clazz, markerType, markerValue);
 618         List<Object[]> daoResults = new ArrayList<>();
 619         if(numberOfResults > 0) { // no point checking again
 620             daoResults = dao.findByMarker(clazz, markerType, markerValue, includeEntity,
 621                     pageSize, pageNumber, propertyPaths);
 622         }
 623
 624         List<MarkedEntityDTO<S>> result = new ArrayList<>();
 625         for (Object[] daoObj : daoResults){
 626             if (includeEntity){
 627                 result.add(new MarkedEntityDTO<S>((MarkerType)daoObj[0], (Boolean)daoObj[1], (S)daoObj[2]));
 628             }else{
 629                 result.add(new MarkedEntityDTO<S>((MarkerType)daoObj[0], (Boolean)daoObj[1], (UUID)daoObj[2], (String)daoObj[3]));
 630             }
 631         }
 632         return new DefaultPagerImpl<MarkedEntityDTO<S>>(pageNumber, numberOfResults, pageSize, result);
 633     }
 634 }
 635