cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IdentifiableServiceBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.api.service;
  11
  12 import java.util.ArrayList;
  13 import java.util.Arrays;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22 import org.hibernate.criterion.Criterion;
  23 import org.springframework.transaction.annotation.Transactional;
  24
  25 import eu.etaxonomy.cdm.api.service.config.IIdentifiableEntityServiceConfigurator;
  26 import eu.etaxonomy.cdm.api.service.dto.IdentifiedEntityDTO;
  27 import eu.etaxonomy.cdm.api.service.dto.MarkedEntityDTO;
  28 import eu.etaxonomy.cdm.api.service.pager.Pager;
  29 import eu.etaxonomy.cdm.api.service.pager.impl.DefaultPagerImpl;
  30 import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
  31 import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
  32 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  33 import eu.etaxonomy.cdm.model.common.CdmBase;
  34 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  35 import eu.etaxonomy.cdm.model.common.ISourceable;
  36 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LSID;
  39 import eu.etaxonomy.cdm.model.common.MarkerType;
  40 import eu.etaxonomy.cdm.model.media.Rights;
  41 import eu.etaxonomy.cdm.model.name.NonViralName;
  42 import eu.etaxonomy.cdm.model.reference.Reference;
  43 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  44 import eu.etaxonomy.cdm.persistence.dao.common.IIdentifiableDao;
  45 import eu.etaxonomy.cdm.persistence.dao.hibernate.HibernateBeanInitializer;
  46 import eu.etaxonomy.cdm.persistence.dao.initializer.AutoPropertyInitializer;
  47 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  48 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  49 import eu.etaxonomy.cdm.persistence.query.OrderHint;
  50 import eu.etaxonomy.cdm.persistence.query.OrderHint.SortOrder;
  51 import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy;
  52 import eu.etaxonomy.cdm.strategy.match.DefaultMatchStrategy;
  53 import eu.etaxonomy.cdm.strategy.match.IMatchStrategy;
  54 import eu.etaxonomy.cdm.strategy.match.IMatchable;
  55 import eu.etaxonomy.cdm.strategy.match.MatchException;
  56 import eu.etaxonomy.cdm.strategy.merge.IMergable;
  57 import eu.etaxonomy.cdm.strategy.merge.IMergeStrategy;
  58 import eu.etaxonomy.cdm.strategy.merge.MergeException;
  59
  60 public abstract class IdentifiableServiceBase<T extends IdentifiableEntity, DAO extends IIdentifiableDao<T>> extends AnnotatableServiceBase<T,DAO>
  61                                                 implements IIdentifiableEntityService<T>{
  62
  63
  64         protected static final int UPDATE_TITLE_CACHE_DEFAULT_STEP_SIZE = 1000;
  65         protected static final  Logger logger = Logger.getLogger(IdentifiableServiceBase.class);
  66
  67         @Override
  68         @Transactional(readOnly = true)
  69         public Pager<Rights> getRights(T t, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
  70         Integer numberOfResults = dao.countRights(t);
  71
  72                 List<Rights> results = new ArrayList<Rights>();
  73                 if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
  74                         results = dao.getRights(t, pageSize, pageNumber,propertyPaths);
  75                 }
  76
  77                 return new DefaultPagerImpl<Rights>(pageNumber, numberOfResults, pageSize, results);
  78         }
  79
  80         @Override
  81         @Transactional(readOnly = true)
  82         public Pager<IdentifiableSource> getSources(T t, Integer pageSize, Integer pageNumber, List<String> propertyPaths) {
  83                  Integer numberOfResults = dao.countSources(t);
  84
  85                  List<IdentifiableSource> results = new ArrayList<IdentifiableSource>();
  86                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
  87                          results = dao.getSources(t, pageSize, pageNumber,propertyPaths);
  88                  }
  89
  90                  return new DefaultPagerImpl<IdentifiableSource>(pageNumber, numberOfResults, pageSize, results);
  91         }
  92
  93
  94         @Transactional(readOnly = false)
  95         @Override
  96         public T replace(T x, T y) {
  97                 return dao.replace(x, y);
  98         }
  99         /**
 100          * FIXME Candidate for harmonization
 101          * Given that this method is strongly typed, and generic, could we not simply expose it as
 102          * List<T> findByTitle(String title) as it is somewhat less cumbersome. Admittedly, I don't
 103          * understand what is going on with the configurators etc. so maybe there is a good reason for
 104          * the design of this method.
 105          * @param title
 106          * @return
 107          */
 108         @Transactional(readOnly = true)
 109         protected List<T> findCdmObjectsByTitle(String title){
 110                 return ((IIdentifiableDao)dao).findByTitle(title);
 111         }
 112
 113         @Transactional(readOnly = true)
 114         protected List<T> findCdmObjectsByTitle(String title, Class<T> clazz){
 115                 return ((IIdentifiableDao)dao).findByTitleAndClass(title, clazz);
 116         }
 117         @Transactional(readOnly = true)
 118         protected List<T> findCdmObjectsByTitle(String title, CdmBase sessionObject){
 119                 return ((IIdentifiableDao)dao).findByTitle(title, sessionObject);
 120         }
 121
 122         /*
 123          * TODO - Migrated from CommonServiceBase
 124          *  (non-Javadoc)
 125          * @see eu.etaxonomy.cdm.api.service.ICommonService#getSourcedObjectById(java.lang.String, java.lang.String)
 126          */
 127         @Transactional(readOnly = true)
 128         @Override
 129         public ISourceable getSourcedObjectByIdInSource(Class clazz, String idInSource, String idNamespace) {
 130                 ISourceable result = null;
 131
 132                 List<T> list = dao.findOriginalSourceByIdInSource(idInSource, idNamespace);
 133                 if (! list.isEmpty()){
 134                         result = list.get(0);
 135                 }
 136                 return result;
 137         }
 138
 139         @Transactional(readOnly = true)
 140         @Override
 141         public List<UuidAndTitleCache<T>> getUuidAndTitleCache(Integer limit, String pattern) {
 142                 return dao.getUuidAndTitleCache(limit, pattern);
 143         }
 144
 145     @Transactional(readOnly = true)
 146     @Override
 147     public <S extends T> List<UuidAndTitleCache<S>> getUuidAndTitleCache(Class<S> clazz,Integer limit, String pattern) {
 148         return dao.getUuidAndTitleCache(clazz, limit, pattern);
 149     }
 150
 151     @Transactional(readOnly = true)
 152     @Override
 153     public String getTitleCache(UUID uuid, boolean refresh){
 154         return dao.getTitleCache(uuid, refresh);
 155     }
 156
 157         @Transactional(readOnly = true)
 158         @Override
 159         public Pager<T> findByTitle(Class<? extends T> clazz, String queryString,MatchMode matchmode, List<Criterion> criteria, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 160                  long numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria);
 161
 162                  List<T> results = new ArrayList<T>();
 163                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 164                                 results = dao.findByTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths);
 165                  }
 166
 167                   return new DefaultPagerImpl<T>(pageNumber, numberOfResults, pageSize, results);
 168         }
 169
 170         @Transactional(readOnly = true)
 171         @Override
 172         public Pager<T> findByTitle(IIdentifiableEntityServiceConfigurator<T> config){
 173                 return findByTitle(config.getClazz(), config.getTitleSearchStringSqlized(), config.getMatchMode(), config.getCriteria(), config.getPageSize(), config.getPageNumber(), config.getOrderHints(), config.getPropertyPaths());
 174         }
 175
 176         @Transactional(readOnly = true)
 177         @Override
 178         public List<T> listByTitle(Class<? extends T> clazz, String queryString,MatchMode matchmode, List<Criterion> criteria, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 179                  long numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria);
 180
 181                  List<T> results = new ArrayList<T>();
 182                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 183                                 results = dao.findByTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths);
 184                  }
 185                  return results;
 186         }
 187
 188         @Transactional(readOnly = true)
 189         @Override
 190         public Pager<T> findTitleCache(Class<? extends T> clazz, String queryString, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, MatchMode matchMode){
 191                 long numberOfResults = dao.countTitleCache(clazz, queryString, matchMode);
 192
 193                  List<T> results = new ArrayList<T>();
 194                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 195                                 results = dao.findTitleCache(clazz, queryString, pageSize, pageNumber, orderHints, matchMode);
 196                  }
 197                  long r = 0;
 198                  r += numberOfResults;
 199
 200                   return new DefaultPagerImpl<T>(pageNumber, r , pageSize, results);
 201         }
 202
 203         @Transactional(readOnly = true)
 204         @Override
 205         public List<T> listByReferenceTitle(Class<? extends T> clazz, String queryString,MatchMode matchmode, List<Criterion> criteria, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 206                  long numberOfResults = dao.countByReferenceTitle(clazz, queryString, matchmode, criteria);
 207
 208                  List<T> results = new ArrayList<T>();
 209                  if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 210                                 results = dao.findByReferenceTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths);
 211                  }
 212                  return results;
 213         }
 214
 215         @Transactional(readOnly = true)
 216         @Override
 217         public T find(LSID lsid) {
 218                 return dao.find(lsid);
 219         }
 220
 221         @Transactional(readOnly = true)
 222         @Override
 223         public Pager<T> search(Class<? extends T> clazz, String queryString, Integer pageSize, Integer pageNumber, List<OrderHint> orderHints, List<String> propertyPaths) {
 224         Integer numberOfResults = dao.count(clazz,queryString);
 225
 226                 List<T> results = new ArrayList<T>();
 227                 if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize)
 228                         results = dao.search(clazz,queryString, pageSize, pageNumber, orderHints, propertyPaths);
 229                 }
 230
 231                 return new DefaultPagerImpl<T>(pageNumber, numberOfResults, pageSize, results);
 232         }
 233
 234         @Override
 235         @Transactional(readOnly = false)
 236         public void updateTitleCache() {
 237                 updateTitleCache(null, null, null, null);
 238         }
 239
 240         @Transactional(readOnly = false)  //TODO check transactional behaviour, e.g. what happens with the session if count is very large
 241         protected <S extends T > void updateTitleCacheImpl(Class<S> clazz, Integer stepSize, IIdentifiableEntityCacheStrategy<T> cacheStrategy, IProgressMonitor monitor) {
 242                 if (stepSize == null){
 243                         stepSize = UPDATE_TITLE_CACHE_DEFAULT_STEP_SIZE;
 244                 }
 245                 if (monitor == null){
 246                         monitor = DefaultProgressMonitor.NewInstance();
 247                 }
 248
 249                 int count = dao.count(clazz);
 250                 monitor.beginTask("update titles", count);
 251                 int worked = 0;
 252                 for(int i = 0 ; i < count ; i = i + stepSize){
 253                         // not sure if such strict ordering is necessary here, but for safety reasons I do it
 254                         ArrayList<OrderHint> orderHints = new ArrayList<OrderHint>();
 255                         orderHints.add( new OrderHint("id", OrderHint.SortOrder.ASCENDING));
 256
 257
 258                         Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> oldAutoInit = switchOfAutoinitializer();
 259                         List<S> list = this.list(clazz, stepSize, i, orderHints, null);
 260                         switchOnOldAutoInitializer(oldAutoInit);
 261
 262                         List<T> entitiesToUpdate = new ArrayList<T>();
 263                         for (T entity : list){
 264                                 HibernateProxyHelper.deproxy(entity, clazz);
 265                                 if (entity.isProtectedTitleCache() == false){
 266                                         updateTitleCacheForSingleEntity(cacheStrategy, entitiesToUpdate, entity);
 267                                 }
 268                                 worked++;
 269                         }
 270                         for (T entity: entitiesToUpdate){
 271                                 if (entity.getTitleCache() != null){
 272                                         //System.err.println(entity.getTitleCache());
 273                                 }else{
 274                                         //System.err.println("no titleCache" + ((NonViralName)entity).getNameCache());
 275                                 }
 276                         }
 277                         saveOrUpdate(entitiesToUpdate);
 278                         monitor.worked(list.size());
 279                         if (monitor.isCanceled()){
 280                                 monitor.done();
 281                                 return;
 282                         }
 283                 }
 284                 monitor.done();
 285         }
 286
 287         /**
 288          * Brings back all auto initializers to the bean initializer
 289          * @see #switchOfAutoinitializer()
 290          * @param oldAutoInit
 291          */
 292         protected void switchOnOldAutoInitializer(
 293                         Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> oldAutoInit) {
 294                 HibernateBeanInitializer initializer = (HibernateBeanInitializer)this.appContext.getBean("defaultBeanInitializer");
 295                 initializer.setBeanAutoInitializers(oldAutoInit);
 296         }
 297
 298         /**
 299          * Removes all auto initializers from the bean initializer
 300          *
 301          * @see #switchOnOldAutoInitializer(Map)
 302          * @return
 303          */
 304         protected Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> switchOfAutoinitializer() {
 305                 HibernateBeanInitializer initializer = (HibernateBeanInitializer)this.appContext.getBean("defaultBeanInitializer");
 306                 Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> oldAutoInitializers = initializer.getBeanAutoInitializers();
 307                 Map<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>> map = new HashMap<Class<? extends CdmBase>, AutoPropertyInitializer<CdmBase>>();
 308                 initializer.setBeanAutoInitializers(map);
 309                 return oldAutoInitializers;
 310         }
 311
 312         /**
 313          * @param cacheStrategy
 314          * @param entitiesToUpdate
 315          * @param entity
 316          */
 317         /**
 318          * @param cacheStrategy
 319          * @param entitiesToUpdate
 320          * @param entity
 321          */
 322         @SuppressWarnings("unchecked")
 323         private void updateTitleCacheForSingleEntity(
 324                         IIdentifiableEntityCacheStrategy<T> cacheStrategy,
 325                         List<T> entitiesToUpdate, T entity) {
 326
 327                 //assert (entity.isProtectedTitleCache() == false );
 328
 329                 //exclude recursive inreferences
 330                 if (entity.isInstanceOf(Reference.class)){
 331                         Reference ref = CdmBase.deproxy(entity, Reference.class);
 332                         if (ref.getInReference() != null && ref.getInReference().equals(ref)){
 333                                 return;
 334                         }
 335                 }
 336
 337
 338                 //define the correct cache strategy
 339                 IIdentifiableEntityCacheStrategy entityCacheStrategy = cacheStrategy;
 340                 if (entityCacheStrategy == null){
 341                         entityCacheStrategy = entity.getCacheStrategy();
 342                         //FIXME find out why the wrong cache strategy is loaded here, see #1876
 343                         if (entity instanceof Reference){
 344                                 entityCacheStrategy = ReferenceFactory.newReference(((Reference)entity).getType()).getCacheStrategy();
 345                         }
 346                 }
 347
 348
 349
 350                 //old titleCache
 351                 entity.setProtectedTitleCache(true);
 352
 353                 String oldTitleCache = entity.getTitleCache();
 354                 entity.setTitleCache(oldTitleCache, false);   //before we had entity.setProtectedTitleCache(false) but this deleted the titleCache itself
 355                 entity.setCacheStrategy(entityCacheStrategy);
 356                 //NonViralNames and Reference have more caches //TODO handle in NameService
 357                 String oldNameCache = null;
 358                 String oldFullTitleCache = null;
 359                 String oldAbbrevTitleCache = null;
 360                 if (entity instanceof NonViralName ){
 361
 362                         try{
 363                                 NonViralName<?> nvn = (NonViralName) entity;
 364                                 if (!nvn.isProtectedNameCache()){
 365                                         nvn.setProtectedNameCache(true);
 366                                         oldNameCache = nvn.getNameCache();
 367                                         nvn.setProtectedNameCache(false);
 368                                 }
 369                                 if (!nvn.isProtectedFullTitleCache()){
 370                                         nvn.setProtectedFullTitleCache(true);
 371                                         oldFullTitleCache = nvn.getFullTitleCache();
 372                                         nvn.setProtectedFullTitleCache(false);
 373                                 }
 374                         }catch(ClassCastException e){
 375                                 System.out.println("entity: " + entity.getTitleCache());
 376                         }
 377
 378                 }else if (entity instanceof Reference){
 379                         Reference ref = (Reference) entity;
 380                         if (!ref.isProtectedAbbrevTitleCache()){
 381                                 ref.setProtectedAbbrevTitleCache(true);
 382                                 oldAbbrevTitleCache = ref.getAbbrevTitleCache();
 383                                 ref.setProtectedAbbrevTitleCache(false);
 384                         }
 385                 }
 386                 setOtherCachesNull(entity);
 387                 String newTitleCache= null;
 388                 NonViralName<?> nvn = null;//TODO find better solution
 389                 try{
 390                         if (entity instanceof NonViralName){
 391                                 nvn = (NonViralName) entity;
 392                                 newTitleCache = entityCacheStrategy.getTitleCache(nvn);
 393                         } else{
 394                                  newTitleCache = entityCacheStrategy.getTitleCache(entity);
 395                         }
 396                 }catch (ClassCastException e){
 397                         nvn = HibernateProxyHelper.deproxy(entity, NonViralName.class);
 398                         newTitleCache = entityCacheStrategy.getTitleCache(nvn);
 399                         //System.out.println("titleCache: " +entity.getTitleCache());
 400                 }
 401
 402                 if ( oldTitleCache == null   || oldTitleCache != null && ! oldTitleCache.equals(newTitleCache) ){
 403                         entity.setTitleCache(null, false);
 404                         String newCache = entity.getTitleCache();
 405
 406                         if (newCache == null){
 407                                 logger.warn("newCache should never be null");
 408                         }
 409                         if (oldTitleCache == null){
 410                                 logger.info("oldTitleCache should never be null");
 411                         }
 412                         if (nvn != null){
 413                                 //NonViralName<?> nvn = (NonViralName) entity;
 414                                 nvn.getNameCache();
 415                                 nvn.getFullTitleCache();
 416                         }
 417                         if (entity instanceof Reference){
 418                                 Reference ref = (Reference) entity;
 419                                 ref.getAbbrevTitleCache();
 420                         }
 421                         entitiesToUpdate.add(entity);
 422                 }else if (nvn != null){
 423                         //NonViralName<?> nvn = (NonViralName) entity;
 424                         String newNameCache = nvn.getNameCache();
 425                         String newFullTitleCache = nvn.getFullTitleCache();
 426                         if ((oldNameCache == null && !nvn.isProtectedNameCache()) || (oldNameCache != null && !oldNameCache.equals(newNameCache))){
 427                                 entitiesToUpdate.add(entity);
 428                         }else if ((oldFullTitleCache == null && !nvn.isProtectedFullTitleCache()) || (oldFullTitleCache != null && !oldFullTitleCache.equals(newFullTitleCache))){
 429                                 entitiesToUpdate.add(entity);
 430                         }
 431                 }else if (entity instanceof Reference){
 432                         Reference ref = (Reference) entity;
 433                         String newAbbrevTitleCache = ref.getAbbrevTitleCache();
 434                         if ( (oldAbbrevTitleCache == null && !ref.isProtectedAbbrevTitleCache() ) || (oldAbbrevTitleCache != null && !oldAbbrevTitleCache.equals(newAbbrevTitleCache))){
 435                                 entitiesToUpdate.add(entity);
 436                         }
 437                 }
 438
 439
 440         }
 441
 442
 443
 444         /**
 445          * Needs override if not only the title cache should be set to null to
 446          * generate the correct new title cache
 447          */
 448         protected void setOtherCachesNull(T entity) {
 449                 return;
 450         }
 451
 452
 453
 454         private class DeduplicateState{
 455                 String lastTitleCache;
 456                 Integer pageSize = 50;
 457                 int nPages = 3;
 458                 int startPage = 0;
 459                 boolean isCompleted = false;
 460                 int result;
 461         }
 462
 463         @Override
 464         @Transactional(readOnly = false)
 465         public int deduplicate(Class<? extends T> clazz, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) {
 466                 DeduplicateState dedupState = new DeduplicateState();
 467
 468                 if (clazz == null){
 469                         logger.warn("Deduplication clazz must not be null!");
 470                         return 0;
 471                 }
 472                 if (! ( IMatchable.class.isAssignableFrom(clazz) && IMergable.class.isAssignableFrom(clazz) )  ){
 473                         logger.warn("Deduplication implemented only for classes implementing IMatchable and IMergeable. No deduplication performed!");
 474                         return 0;
 475                 }
 476                 Class matchableClass = clazz;
 477                 if (matchStrategy == null){
 478                         matchStrategy = DefaultMatchStrategy.NewInstance(matchableClass);
 479                 }
 480                 List<T> nextGroup = new ArrayList<T>();
 481
 482                 int result = 0;
 483 //              double countTotal = count(clazz);
 484 //
 485 //              Number countPagesN = Math.ceil(countTotal/dedupState.pageSize.doubleValue()) ;
 486 //              int countPages = countPagesN.intValue();
 487 //
 488
 489                 List<OrderHint> orderHints = Arrays.asList(new OrderHint[]{new OrderHint("titleCache", SortOrder.ASCENDING)});
 490
 491                 while (! dedupState.isCompleted){
 492                         //get x page sizes
 493                         List<T> objectList = getPages(clazz, dedupState, orderHints);
 494                         //after each page check if any changes took place
 495                         int nUnEqualPages = handleAllPages(objectList, dedupState, nextGroup, matchStrategy, mergeStrategy);
 496                         nUnEqualPages = nUnEqualPages + dedupState.pageSize * dedupState.startPage;
 497                         //refresh start page counter
 498                         int finishedPages = nUnEqualPages / dedupState.pageSize;
 499                         dedupState.startPage = finishedPages;
 500                 }
 501
 502                 result += handleLastGroup(nextGroup, matchStrategy, mergeStrategy);
 503                 return result;
 504         }
 505
 506
 507         private int handleAllPages(List<T> objectList, DeduplicateState dedupState, List<T> nextGroup, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) {
 508                 int nUnEqual = 0;
 509                 for (T object : objectList){
 510                         String currentTitleCache = object.getTitleCache();
 511                         if (currentTitleCache != null && currentTitleCache.equals(dedupState.lastTitleCache)){
 512                                 //=titleCache
 513                                 nextGroup.add(object);
 514                         }else{
 515                                 //<> titleCache
 516                                 dedupState.result += handleLastGroup(nextGroup, matchStrategy, mergeStrategy);
 517                                 nextGroup = new ArrayList<T>();
 518                                 nextGroup.add(object);
 519                                 nUnEqual++;
 520                         }
 521                         dedupState.lastTitleCache = currentTitleCache;
 522                 }
 523                 handleLastGroup(nextGroup, matchStrategy, mergeStrategy);
 524                 return nUnEqual;
 525         }
 526
 527         private List<T> getPages(Class<? extends T> clazz, DeduplicateState dedupState, List<OrderHint> orderHints) {
 528                 List<T> result = new ArrayList<T>();
 529                 for (int pageNo = dedupState.startPage; pageNo < dedupState.startPage + dedupState.nPages; pageNo++){
 530                         List<T> objectList = listByTitle(clazz, null, null, null, dedupState.pageSize, pageNo, orderHints, null);
 531                         result.addAll(objectList);
 532                 }
 533                 if (result.size()< dedupState.nPages * dedupState.pageSize ){
 534                         dedupState.isCompleted = true;
 535                 }
 536                 return result;
 537         }
 538
 539         private int handleLastGroup(List<T> group, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) {
 540                 int result = 0;
 541                 int size = group.size();
 542                 Set<Integer> exclude = new HashSet<Integer>();  //set to collect all objects, that have been merged already
 543                 for (int i = 0; i < size - 1; i++){
 544                         if (exclude.contains(i)){
 545                                 continue;
 546                         }
 547                         for (int j = i + 1; j < size; j++){
 548                                 if (exclude.contains(j)){
 549                                         continue;
 550                                 }
 551                                 T firstObject = group.get(i);
 552                                 T secondObject = group.get(j);
 553
 554                                 try {
 555                                         if (matchStrategy.invoke((IMatchable)firstObject, (IMatchable)secondObject)){
 556                                                 commonService.merge((IMergable)firstObject, (IMergable)secondObject, mergeStrategy);
 557                                                 exclude.add(j);
 558                                                 result++;
 559                                         }
 560                                 } catch (MatchException e) {
 561                                         logger.warn("MatchException when trying to match " + firstObject.getTitleCache());
 562                                         e.printStackTrace();
 563                                 } catch (MergeException e) {
 564                                         logger.warn("MergeException when trying to merge " + firstObject.getTitleCache());
 565                                         e.printStackTrace();
 566                                 }
 567                         }
 568                 }
 569                 return result;
 570         }
 571
 572         @Transactional(readOnly = true)
 573         @Override
 574         public Integer countByTitle(Class<? extends T> clazz, String queryString,MatchMode matchmode, List<Criterion> criteria){
 575                  long numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria);
 576
 577                  return ((Number)numberOfResults).intValue();
 578         }
 579
 580         @Transactional(readOnly = true)
 581         @Override
 582         public Integer countByTitle(IIdentifiableEntityServiceConfigurator<T> config){
 583                 return countByTitle(config.getClazz(), config.getTitleSearchStringSqlized(),
 584                                 config.getMatchMode(), config.getCriteria());
 585
 586         }
 587
 588         @Override
 589         @Transactional(readOnly = true)
 590         public <S extends T> Pager<IdentifiedEntityDTO<S>> findByIdentifier(
 591                         Class<S> clazz, String identifier, DefinedTerm identifierType, MatchMode matchmode,
 592                         boolean includeEntity, Integer pageSize,
 593                         Integer pageNumber,     List<String> propertyPaths) {
 594
 595                 Integer numberOfResults = dao.countByIdentifier(clazz, identifier, identifierType, matchmode);
 596         List<Object[]> daoResults = new ArrayList<Object[]>();
 597         if(numberOfResults > 0) { // no point checking again
 598                 daoResults = dao.findByIdentifier(clazz, identifier, identifierType,
 599                                 matchmode, includeEntity, pageSize, pageNumber, propertyPaths);
 600         }
 601
 602         List<IdentifiedEntityDTO<S>> result = new ArrayList<IdentifiedEntityDTO<S>>();
 603         for (Object[] daoObj : daoResults){
 604                 if (includeEntity){
 605                         result.add(new IdentifiedEntityDTO<S>((DefinedTerm)daoObj[0], (String)daoObj[1], (S)daoObj[2]));
 606                 }else{
 607                         result.add(new IdentifiedEntityDTO<S>((DefinedTerm)daoObj[0], (String)daoObj[1], (UUID)daoObj[2], (String)daoObj[3]));
 608                 }
 609         }
 610                 return new DefaultPagerImpl<IdentifiedEntityDTO<S>>(pageNumber, numberOfResults, pageSize, result);
 611         }
 612
 613     @Override
 614     @Transactional(readOnly = true)
 615     public <S extends T> Pager<MarkedEntityDTO<S>> findByMarker(
 616             Class<S> clazz, MarkerType markerType, Boolean markerValue,
 617             boolean includeEntity, Integer pageSize,
 618             Integer pageNumber, List<String> propertyPaths) {
 619
 620         Long numberOfResults = dao.countByMarker(clazz, markerType, markerValue);
 621         List<Object[]> daoResults = new ArrayList<>();
 622         if(numberOfResults > 0) { // no point checking again
 623             daoResults = dao.findByMarker(clazz, markerType, markerValue, includeEntity,
 624                     pageSize, pageNumber, propertyPaths);
 625         }
 626
 627         List<MarkedEntityDTO<S>> result = new ArrayList<>();
 628         for (Object[] daoObj : daoResults){
 629             if (includeEntity){
 630                 result.add(new MarkedEntityDTO<S>((MarkerType)daoObj[0], (Boolean)daoObj[1], (S)daoObj[2]));
 631             }else{
 632                 result.add(new MarkedEntityDTO<S>((MarkerType)daoObj[0], (Boolean)daoObj[1], (UUID)daoObj[2], (String)daoObj[3]));
 633             }
 634         }
 635         return new DefaultPagerImpl<MarkedEntityDTO<S>>(pageNumber, numberOfResults, pageSize, result);
 636     }
 637 }
 638