X-Git-Url: https://dev.e-taxonomy.eu/gitweb/cdmlib.git/blobdiff_plain/f57f9ab629a27e8fe8f2a961d003a061285c7cad..6509418e98662e6311710bad9cc87046c102eac7:/cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IdentifiableServiceBase.java diff --git a/cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IdentifiableServiceBase.java b/cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IdentifiableServiceBase.java index de8d0bb42d..8d0d728cf4 100644 --- a/cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IdentifiableServiceBase.java +++ b/cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IdentifiableServiceBase.java @@ -11,7 +11,12 @@ package eu.etaxonomy.cdm.api.service; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.log4j.Logger; import org.hibernate.criterion.Criterion; @@ -20,6 +25,9 @@ import org.springframework.transaction.annotation.Transactional; import eu.etaxonomy.cdm.api.service.config.IIdentifiableEntityServiceConfigurator; import eu.etaxonomy.cdm.api.service.pager.Pager; import eu.etaxonomy.cdm.api.service.pager.impl.DefaultPagerImpl; +import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor; +import eu.etaxonomy.cdm.common.monitor.IProgressMonitor; +import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.common.ISourceable; import eu.etaxonomy.cdm.model.common.IdentifiableEntity; @@ -27,51 +35,60 @@ import eu.etaxonomy.cdm.model.common.IdentifiableSource; import eu.etaxonomy.cdm.model.common.LSID; import eu.etaxonomy.cdm.model.common.UuidAndTitleCache; import eu.etaxonomy.cdm.model.media.Rights; -import eu.etaxonomy.cdm.model.reference.ReferenceBase; +import eu.etaxonomy.cdm.model.name.NonViralName; +import eu.etaxonomy.cdm.model.reference.Reference; import eu.etaxonomy.cdm.model.reference.ReferenceFactory; import eu.etaxonomy.cdm.persistence.dao.common.IIdentifiableDao; +import eu.etaxonomy.cdm.persistence.dao.hibernate.HibernateBeanInitializer; +import eu.etaxonomy.cdm.persistence.dao.initializer.AutoPropertyInitializer; import eu.etaxonomy.cdm.persistence.query.MatchMode; import eu.etaxonomy.cdm.persistence.query.OrderHint; +import eu.etaxonomy.cdm.persistence.query.OrderHint.SortOrder; import eu.etaxonomy.cdm.strategy.cache.common.IIdentifiableEntityCacheStrategy; +import eu.etaxonomy.cdm.strategy.match.DefaultMatchStrategy; +import eu.etaxonomy.cdm.strategy.match.IMatchStrategy; +import eu.etaxonomy.cdm.strategy.match.IMatchable; +import eu.etaxonomy.cdm.strategy.match.MatchException; +import eu.etaxonomy.cdm.strategy.merge.IMergable; +import eu.etaxonomy.cdm.strategy.merge.IMergeStrategy; +import eu.etaxonomy.cdm.strategy.merge.MergeException; public abstract class IdentifiableServiceBase> extends AnnotatableServiceBase implements IIdentifiableEntityService{ + protected static final int UPDATE_TITLE_CACHE_DEFAULT_STEP_SIZE = 1000; protected static final Logger logger = Logger.getLogger(IdentifiableServiceBase.class); + @Override @Transactional(readOnly = true) public Pager getRights(T t, Integer pageSize, Integer pageNumber, List propertyPaths) { Integer numberOfResults = dao.countRights(t); List results = new ArrayList(); - if(numberOfResults > 0) { // no point checking again + if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize) results = dao.getRights(t, pageSize, pageNumber,propertyPaths); } return new DefaultPagerImpl(pageNumber, numberOfResults, pageSize, results); } + @Override @Transactional(readOnly = true) public Pager getSources(T t, Integer pageSize, Integer pageNumber, List propertyPaths) { Integer numberOfResults = dao.countSources(t); List results = new ArrayList(); - if(numberOfResults > 0) { // no point checking again + if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize) results = dao.getSources(t, pageSize, pageNumber,propertyPaths); } return new DefaultPagerImpl(pageNumber, numberOfResults, pageSize, results); } - @Transactional(readOnly = true) - protected List findByTitle(IIdentifiableEntityServiceConfigurator config){ - return ((IIdentifiableDao)dao).findByTitle(config.getTitleSearchString(), - config.getMatchMode(), 0, -1, null); - // TODO: Implement parameters pageSize, pageNumber, and criteria - } @Transactional(readOnly = false) + @Override public T replace(T x, T y) { return dao.replace(x, y); } @@ -104,6 +121,7 @@ public abstract class IdentifiableServiceBase> getUuidAndTitleCache() { return dao.getUuidAndTitleCache(); } @Transactional(readOnly = true) + @Override public Pager findByTitle(Class clazz, String queryString,MatchMode matchmode, List criteria, Integer pageSize, Integer pageNumber, List orderHints, List propertyPaths) { Integer numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria); List results = new ArrayList(); - if(numberOfResults > 0) { // no point checking again + if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize) results = dao.findByTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths); } @@ -135,68 +152,275 @@ public abstract class IdentifiableServiceBase findByTitle(IIdentifiableEntityServiceConfigurator config){ + return findByTitle(config.getClazz(), config.getTitleSearchStringSqlized(), config.getMatchMode(), config.getCriteria(), config.getPageSize(), config.getPageNumber(), config.getOrderHints(), config.getPropertyPaths()); + } + + @Transactional(readOnly = true) + @Override + public List listByTitle(Class clazz, String queryString,MatchMode matchmode, List criteria, Integer pageSize, Integer pageNumber, List orderHints, List propertyPaths) { + Integer numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria); + + List results = new ArrayList(); + if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize) + results = dao.findByTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths); + } + return results; + } + + @Transactional(readOnly = true) + @Override + public Pager findTitleCache(Class clazz, String queryString, Integer pageSize, Integer pageNumber, List orderHints, MatchMode matchMode){ + long numberOfResults = dao.countTitleCache(clazz, queryString, matchMode); + + List results = new ArrayList(); + if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize) + results = dao.findTitleCache(clazz, queryString, pageSize, pageNumber, orderHints, matchMode); + } + int r = 0; + r += numberOfResults; + + return new DefaultPagerImpl(pageNumber, r , pageSize, results); + } + + @Transactional(readOnly = true) + @Override + public List listByReferenceTitle(Class clazz, String queryString,MatchMode matchmode, List criteria, Integer pageSize, Integer pageNumber, List orderHints, List propertyPaths) { + Integer numberOfResults = dao.countByReferenceTitle(clazz, queryString, matchmode, criteria); + + List results = new ArrayList(); + if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize) + results = dao.findByReferenceTitle(clazz, queryString, matchmode, criteria, pageSize, pageNumber, orderHints, propertyPaths); + } + return results; + } + + @Transactional(readOnly = true) + @Override public T find(LSID lsid) { return dao.find(lsid); } @Transactional(readOnly = true) + @Override public Pager search(Class clazz, String queryString, Integer pageSize, Integer pageNumber, List orderHints, List propertyPaths) { Integer numberOfResults = dao.count(clazz,queryString); List results = new ArrayList(); - if(numberOfResults > 0) { // no point checking again + if(numberOfResults > 0) { // no point checking again //TODO use AbstractPagerImpl.hasResultsInRange(numberOfResults, pageNumber, pageSize) results = dao.search(clazz,queryString, pageSize, pageNumber, orderHints, propertyPaths); } return new DefaultPagerImpl(pageNumber, numberOfResults, pageSize, results); } + @Override @Transactional(readOnly = false) - public void updateTitleCache(Class clazz) { - IIdentifiableEntityCacheStrategy cacheStrategy = null; - updateTitleCache(clazz, UPDATE_TITLE_CACHE_DEFAULT_STEP_SIZE, cacheStrategy); + public void updateTitleCache() { + updateTitleCache(null, null, null, null); } - @Transactional(readOnly = false) //TODO check transactional behaviour, e.g. what happens with the session if count is very large - public void updateTitleCache(Class clazz, Integer stepSize, IIdentifiableEntityCacheStrategy cacheStrategy) { + protected void updateTitleCacheImpl(Class clazz, Integer stepSize, IIdentifiableEntityCacheStrategy cacheStrategy, IProgressMonitor monitor) { if (stepSize == null){ stepSize = UPDATE_TITLE_CACHE_DEFAULT_STEP_SIZE; } - + if (monitor == null){ + monitor = DefaultProgressMonitor.NewInstance(); + } + int count = dao.count(clazz); + monitor.beginTask("update titles", count); + int worked = 0; for(int i = 0 ; i < count ; i = i + stepSize){ // not sure if such strict ordering is necessary here, but for safety reasons I do it ArrayList orderHints = new ArrayList(); orderHints.add( new OrderHint("id", OrderHint.SortOrder.ASCENDING)); - List list = this.list(clazz, stepSize, i, orderHints, null); + + + Map, AutoPropertyInitializer> oldAutoInit = switchOfAutoinitializer(); + List list = this.list(clazz, stepSize, i, orderHints, null); + switchOnOldAutoInitializer(oldAutoInit); + List entitiesToUpdate = new ArrayList(); for (T entity : list){ + HibernateProxyHelper.deproxy(entity, clazz); if (entity.isProtectedTitleCache() == false){ - IIdentifiableEntityCacheStrategy entityCacheStrategy = cacheStrategy; - if (entityCacheStrategy == null){ - entityCacheStrategy = entity.getCacheStrategy(); - //FIXME find out why the wrong cache strategy is loaded here, see #1876 - if (entity instanceof ReferenceBase){ - entityCacheStrategy = ReferenceFactory.newReference(((ReferenceBase)entity).getType()).getCacheStrategy(); - } - } - entity.setCacheStrategy(entityCacheStrategy); - //TODO this won't work for those classes that always generate the title cache new - String titleCache = entity.getTitleCache(); - setOtherCachesNull(entity); //TODO find better solution - String newTitleCache = entityCacheStrategy.getTitleCache(entity); - if (titleCache == null || titleCache != null && ! titleCache.equals(newTitleCache)){ - entity.setTitleCache(null, false); - entity.getTitleCache(); - entitiesToUpdate.add(entity); - } + updateTitleCacheForSingleEntity(cacheStrategy, entitiesToUpdate, entity); + } + worked++; + } + for (T entity: entitiesToUpdate){ + if (entity.getTitleCache() != null){ + //System.err.println(entity.getTitleCache()); + }else{ + //System.err.println("no titleCache" + ((NonViralName)entity).getNameCache()); + } + } + saveOrUpdate(entitiesToUpdate); + monitor.worked(list.size()); + if (monitor.isCanceled()){ + monitor.done(); + return; + } + } + monitor.done(); + } + + /** + * Brings back all auto initializers to the bean initializer + * @see #switchOfAutoinitializer() + * @param oldAutoInit + */ + protected void switchOnOldAutoInitializer( + Map, AutoPropertyInitializer> oldAutoInit) { + HibernateBeanInitializer initializer = (HibernateBeanInitializer)this.appContext.getBean("defaultBeanInitializer"); + initializer.setBeanAutoInitializers(oldAutoInit); + } + + /** + * Removes all auto initializers from the bean initializer + * + * @see #switchOnOldAutoInitializer(Map) + * @return + */ + protected Map, AutoPropertyInitializer> switchOfAutoinitializer() { + HibernateBeanInitializer initializer = (HibernateBeanInitializer)this.appContext.getBean("defaultBeanInitializer"); + Map, AutoPropertyInitializer> oldAutoInitializers = initializer.getBeanAutoInitializers(); + Map, AutoPropertyInitializer> map = new HashMap, AutoPropertyInitializer>(); + initializer.setBeanAutoInitializers(map); + return oldAutoInitializers; + } + + /** + * @param cacheStrategy + * @param entitiesToUpdate + * @param entity + */ + /** + * @param cacheStrategy + * @param entitiesToUpdate + * @param entity + */ + private void updateTitleCacheForSingleEntity( + IIdentifiableEntityCacheStrategy cacheStrategy, + List entitiesToUpdate, T entity) { + + assert (entity.isProtectedTitleCache() == false ); + + //exclude recursive inreferences + if (entity.isInstanceOf(Reference.class)){ + Reference ref = CdmBase.deproxy(entity, Reference.class); + if (ref.getInReference() != null && ref.getInReference().equals(ref)){ + return; + } + } + + + //define the correct cache strategy + IIdentifiableEntityCacheStrategy entityCacheStrategy = cacheStrategy; + if (entityCacheStrategy == null){ + entityCacheStrategy = entity.getCacheStrategy(); + //FIXME find out why the wrong cache strategy is loaded here, see #1876 + if (entity instanceof Reference){ + entityCacheStrategy = ReferenceFactory.newReference(((Reference)entity).getType()).getCacheStrategy(); + } + } + entity.setCacheStrategy(entityCacheStrategy); + + + //old titleCache + entity.setProtectedTitleCache(true); + String oldTitleCache = entity.getTitleCache(); + entity.setTitleCache(oldTitleCache, false); //before we had entity.setProtectedTitleCache(false) but this deleted the titleCache itself + + //NonViralNames and Reference have more caches //TODO handle in NameService + String oldNameCache = null; + String oldFullTitleCache = null; + String oldAbbrevTitleCache = null; + if (entity instanceof NonViralName ){ + + try{ + NonViralName nvn = (NonViralName) entity; + if (!nvn.isProtectedNameCache()){ + nvn.setProtectedNameCache(true); + oldNameCache = nvn.getNameCache(); + nvn.setProtectedNameCache(false); } + if (!nvn.isProtectedFullTitleCache()){ + nvn.setProtectedFullTitleCache(true); + oldFullTitleCache = nvn.getFullTitleCache(); + nvn.setProtectedFullTitleCache(false); + } + }catch(ClassCastException e){ + System.out.println("entity: " + entity.getTitleCache()); } - saveOrUpdateAll(entitiesToUpdate); + }else if (entity instanceof Reference){ + Reference ref = (Reference) entity; + if (!ref.isProtectedAbbrevTitleCache()){ + ref.setProtectedAbbrevTitleCache(true); + oldAbbrevTitleCache = ref.getAbbrevTitleCache(); + ref.setProtectedAbbrevTitleCache(false); + } + } + setOtherCachesNull(entity); + String newTitleCache= null; + NonViralName nvn = null;//TODO find better solution + try{ + if (entity instanceof NonViralName){ + nvn = (NonViralName) entity; + newTitleCache = entityCacheStrategy.getTitleCache(nvn); + } else{ + newTitleCache = entityCacheStrategy.getTitleCache(entity); + } + }catch (ClassCastException e){ + nvn = HibernateProxyHelper.deproxy(entity, NonViralName.class); + newTitleCache = entityCacheStrategy.getTitleCache(nvn); + //System.out.println("titleCache: " +entity.getTitleCache()); + } + + if ( oldTitleCache == null || oldTitleCache != null && ! oldTitleCache.equals(newTitleCache) ){ + entity.setTitleCache(null, false); + String newCache = entity.getTitleCache(); + if (newCache == null){ + logger.warn("newCache should never be null"); + } + if (oldTitleCache == null){ + logger.info("oldTitleCache should never be null"); + } + if (nvn != null){ + //NonViralName nvn = (NonViralName) entity; + nvn.getNameCache(); + nvn.getFullTitleCache(); + } + if (entity instanceof Reference){ + Reference ref = (Reference) entity; + ref.getAbbrevTitleCache(); + } + entitiesToUpdate.add(entity); + }else if (nvn != null){ + //NonViralName nvn = (NonViralName) entity; + String newNameCache = nvn.getNameCache(); + String newFullTitleCache = nvn.getFullTitleCache(); + if ((oldNameCache == null && !nvn.isProtectedNameCache()) || (oldNameCache != null && !oldNameCache.equals(newNameCache))){ + entitiesToUpdate.add(entity); + }else if ((oldFullTitleCache == null && !nvn.isProtectedFullTitleCache()) || (oldFullTitleCache != null && !oldFullTitleCache.equals(newFullTitleCache))){ + entitiesToUpdate.add(entity); + } + }else if (entity instanceof Reference){ + Reference ref = (Reference) entity; + String newAbbrevTitleCache = ref.getAbbrevTitleCache(); + if ( (oldAbbrevTitleCache == null && !ref.isProtectedAbbrevTitleCache() ) || (oldAbbrevTitleCache != null && !oldAbbrevTitleCache.equals(newAbbrevTitleCache))){ + entitiesToUpdate.add(entity); + } } + + } + + /** * Needs override if not only the title cache should be set to null to @@ -205,5 +429,145 @@ public abstract class IdentifiableServiceBase clazz, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) { + DeduplicateState dedupState = new DeduplicateState(); + + if (clazz == null){ + logger.warn("Deduplication clazz must not be null!"); + return 0; + } + if (! ( IMatchable.class.isAssignableFrom(clazz) && IMergable.class.isAssignableFrom(clazz) ) ){ + logger.warn("Deduplication implemented only for classes implementing IMatchable and IMergeable. No deduplication performed!"); + return 0; + } + Class matchableClass = clazz; + if (matchStrategy == null){ + matchStrategy = DefaultMatchStrategy.NewInstance(matchableClass); + } + List nextGroup = new ArrayList(); + + int result = 0; +// double countTotal = count(clazz); +// +// Number countPagesN = Math.ceil(countTotal/dedupState.pageSize.doubleValue()) ; +// int countPages = countPagesN.intValue(); +// + + List orderHints = Arrays.asList(new OrderHint[]{new OrderHint("titleCache", SortOrder.ASCENDING)}); + + while (! dedupState.isCompleted){ + //get x page sizes + List objectList = getPages(clazz, dedupState, orderHints); + //after each page check if any changes took place + int nUnEqualPages = handleAllPages(objectList, dedupState, nextGroup, matchStrategy, mergeStrategy); + nUnEqualPages = nUnEqualPages + dedupState.pageSize * dedupState.startPage; + //refresh start page counter + int finishedPages = nUnEqualPages / dedupState.pageSize; + dedupState.startPage = finishedPages; + } + + result += handleLastGroup(nextGroup, matchStrategy, mergeStrategy); + return result; + } + + + private int handleAllPages(List objectList, DeduplicateState dedupState, List nextGroup, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) { + int nUnEqual = 0; + for (T object : objectList){ + String currentTitleCache = object.getTitleCache(); + if (currentTitleCache != null && currentTitleCache.equals(dedupState.lastTitleCache)){ + //=titleCache + nextGroup.add(object); + }else{ + //<> titleCache + dedupState.result += handleLastGroup(nextGroup, matchStrategy, mergeStrategy); + nextGroup = new ArrayList(); + nextGroup.add(object); + nUnEqual++; + } + dedupState.lastTitleCache = currentTitleCache; + } + handleLastGroup(nextGroup, matchStrategy, mergeStrategy); + return nUnEqual; + } + + private List getPages(Class clazz, DeduplicateState dedupState, List orderHints) { + List result = new ArrayList(); + for (int pageNo = dedupState.startPage; pageNo < dedupState.startPage + dedupState.nPages; pageNo++){ + List objectList = listByTitle(clazz, null, null, null, dedupState.pageSize, pageNo, orderHints, null); + result.addAll(objectList); + } + if (result.size()< dedupState.nPages * dedupState.pageSize ){ + dedupState.isCompleted = true; + } + return result; + } + + private int handleLastGroup(List group, IMatchStrategy matchStrategy, IMergeStrategy mergeStrategy) { + int result = 0; + int size = group.size(); + Set exclude = new HashSet(); //set to collect all objects, that have been merged already + for (int i = 0; i < size - 1; i++){ + if (exclude.contains(i)){ + continue; + } + for (int j = i + 1; j < size; j++){ + if (exclude.contains(j)){ + continue; + } + T firstObject = group.get(i); + T secondObject = group.get(j); + + try { + if (matchStrategy.invoke((IMatchable)firstObject, (IMatchable)secondObject)){ + commonService.merge((IMergable)firstObject, (IMergable)secondObject, mergeStrategy); + exclude.add(j); + result++; + } + } catch (MatchException e) { + logger.warn("MatchException when trying to match " + firstObject.getTitleCache()); + e.printStackTrace(); + } catch (MergeException e) { + logger.warn("MergeException when trying to merge " + firstObject.getTitleCache()); + e.printStackTrace(); + } + } + } + return result; + } + + @Transactional(readOnly = true) + @Override + public Integer countByTitle(Class clazz, String queryString,MatchMode matchmode, List criteria){ + Integer numberOfResults = dao.countByTitle(clazz, queryString, matchmode, criteria); + + return numberOfResults; + } + + @Transactional(readOnly = true) + @Override + public Integer countByTitle(IIdentifiableEntityServiceConfigurator config){ + return countByTitle(config.getClazz(), config.getTitleSearchStringSqlized(), + config.getMatchMode(), config.getCriteria()); + + } + + + + }