EDIT

/**

* Copyright (C) 2017 EDIT

* European Distributed Institute of Taxonomy

* http://www.e-taxonomy.eu

*

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.io.common.utils;

import java.util.Arrays;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import java.util.Map;

import java.util.Optional;

import java.util.Set;

import java.util.UUID;

import java.util.function.Predicate;

import org.apache.log4j.Logger;

import eu.etaxonomy.cdm.api.application.ICdmRepository;

import eu.etaxonomy.cdm.api.service.IService;

import eu.etaxonomy.cdm.common.CdmUtils;

import eu.etaxonomy.cdm.io.common.ImportResult;

import eu.etaxonomy.cdm.io.common.ImportStateBase;

import eu.etaxonomy.cdm.model.agent.AgentBase;

import eu.etaxonomy.cdm.model.agent.Institution;

import eu.etaxonomy.cdm.model.agent.Person;

import eu.etaxonomy.cdm.model.agent.Team;

import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;

import eu.etaxonomy.cdm.model.common.CdmBase;

import eu.etaxonomy.cdm.model.common.ICdmBase;

import eu.etaxonomy.cdm.model.common.IdentifiableEntity;

import eu.etaxonomy.cdm.model.media.Rights;

import eu.etaxonomy.cdm.model.media.RightsType;

import eu.etaxonomy.cdm.model.name.HybridRelationship;

import eu.etaxonomy.cdm.model.name.INonViralName;

import eu.etaxonomy.cdm.model.name.TaxonName;

import eu.etaxonomy.cdm.model.occurrence.Collection;

import eu.etaxonomy.cdm.model.reference.INomenclaturalReference;

import eu.etaxonomy.cdm.model.reference.Reference;

import eu.etaxonomy.cdm.strategy.match.DefaultMatchStrategy;

import eu.etaxonomy.cdm.strategy.match.IMatchStrategyEqual;

import eu.etaxonomy.cdm.strategy.match.IMatchable;

import eu.etaxonomy.cdm.strategy.match.MatchException;

import eu.etaxonomy.cdm.strategy.match.MatchMode;

/**

 * Helper class for deduplicating authors, references, names, etc.

 * during import.

 *

 * Note 2021: Was originally used as fast deduplication tool for commandline imports

 * into empty databases. Currently it is transformed into a deduplication tool that

 * can be used during application based imports.

 *

 * @author a.mueller

 * @since 11.02.2017

 */

public class ImportDeduplicationHelper {

    private static final Logger logger = Logger.getLogger(ImportDeduplicationHelper.class);

    private ICdmRepository repository;

    //for possible future use

    @SuppressWarnings("unused")

    private ImportStateBase<?,?> state;

    public static final int NEVER_USE_MAP = 0;

    public static final int ALWAYS_USE_MAP = -1;

    //should deduplication use maps indexing the full database content? If yes, what is the maximum number of records for this.

    //If more records exist deduplication is done on the fly.

    //0 = never use map

    //-1 = always use map

    private int maxCountFullLoad = ALWAYS_USE_MAP;

    public int getMaxCountFullLoad() {

        return maxCountFullLoad;

    }

    public void setMaxCountFullLoad(int maxCountFullLoad) {

        this.maxCountFullLoad = maxCountFullLoad;

    }

    private enum Status{

        NOT_INIT,

        USE_MAP,

        USE_REPO;

    }

    private class DedupInfo<S extends IdentifiableEntity>{

        Class<S> clazz;

        IMatchStrategyEqual matcher;

        Map<String, Set<S>> map = new HashMap<>();

        Status status = Status.NOT_INIT;

        @SuppressWarnings("unchecked")

        private DedupInfo(Class<S> clazz, DedupMap dedupMap){

            this.clazz = clazz;

            if (IMatchable.class.isAssignableFrom(clazz)) {

                matcher = DefaultMatchStrategy.NewInstance((Class<IMatchable>)clazz);

            }

            dedupMap.put(clazz, this);

        }

        @Override

        public String toString() {

            return clazz.getSimpleName() + ":" + status.name()+":mapsize=" + map.size()+":"+ (matcher == null?"without":"with") + " matcher";

        }

    }

    private class DedupMap<T extends IdentifiableEntity> extends HashMap<Class<T>, DedupInfo<T>>{

        private static final long serialVersionUID = 3757206594833330646L;

    }

    private DedupMap<? extends IdentifiableEntity> dedupMap = new DedupMap<>();

    private DedupInfo<Reference> referenceDedupInfo = new DedupInfo<>(Reference.class, dedupMap);

    private DedupInfo<Person> personDedupInfo = new DedupInfo<>(Person.class, dedupMap);

    private DedupInfo<Team> teamDedupInfo = new DedupInfo<>(Team.class, dedupMap);

    private DedupInfo<TaxonName> nameDedupInfo = new DedupInfo<>(TaxonName.class, dedupMap);

    private Status institutionStatus = Status.NOT_INIT;

    private Status copyrightStatus = Status.NOT_INIT;

    private Status collectionStatus = Status.NOT_INIT;

    private Map<String, Set<Institution>> institutionMap = new HashMap<>();

    //using titleCache

    private Map<String, Set<Rights>> copyrightMap = new HashMap<>();

    private Map<String, Set<Collection>> collectionMap = new HashMap<>();

    /**

     * Clears all internal maps.

     */

    public void reset() {

        dedupMap.values().forEach(di->di.map.clear());

        institutionMap.clear();

        copyrightMap.clear();

        collectionMap.clear();

    }

//    private IMatchStrategy collectionMatcher = DefaultMatchStrategy.NewInstance(Collection.class);

 // ************************** FACTORY *******************************/

     public static <STATE extends ImportStateBase<?,?>> ImportDeduplicationHelper NewInstance(ICdmRepository repository, STATE state){

         return new ImportDeduplicationHelper(repository, state);

     }

 // ************************ CONSTRUCTOR *****************************/

    private ImportDeduplicationHelper(ICdmRepository repository, ImportStateBase<?,?> state) {

         this.repository = repository;

         if (repository == null){

             logger.warn("Repository is null. Deduplication does not work against database.");

         }

         if (state == null){

             logger.warn("State is null. Deduplication works without state.");

         }

         this.state = state;

         try {

             dedupMap.get(Reference.class).matcher.setMatchMode("title", MatchMode.EQUAL);

             dedupMap.get(Team.class).matcher.setMatchMode("nomenclaturalTitleCache", MatchMode.EQUAL_OR_SECOND_NULL);

         } catch (MatchException e) {

             throw new RuntimeException(e);  //should not happen

         }

    }

    public void restartSession(){

        restartSession(repository, null);

    }

    /**

     * Clears all internal maps and loads them with same data as before but in current session.

     */

    public void restartSession(ICdmRepository repository, ImportResult importResult){

        if (repository == null){

            return;

        }

        referenceDedupInfo.map = refreshSetMap(referenceDedupInfo.map, (IService)repository.getReferenceService(), importResult);

        personDedupInfo.map = refreshSetMap(personDedupInfo.map, (IService)repository.getAgentService(), importResult);

        teamDedupInfo.map = refreshSetMap(teamDedupInfo.map, (IService)repository.getAgentService(), importResult);

        institutionMap = refreshSetMap(institutionMap, (IService)repository.getAgentService(), importResult);

        nameDedupInfo.map = refreshSetMap(nameDedupInfo.map, (IService)repository.getNameService(), importResult);

        collectionMap = refreshSetMap(collectionMap, (IService)repository.getCollectionService(), importResult);

        //TODO copyright ?

    }

    //maybe this was used for Institution before

    private <T extends ICdmBase> Map<String, T> refreshMap(Map<String, T> oldMap,

            IService<T> service, ImportResult importResult) {

        Map<String, T> newMap = new HashMap<>();

        for (String key : oldMap.keySet()){

            T old = oldMap.get(key);

            if (old!= null){

                T cdmBase = service.find(old.getUuid());

                if (cdmBase == null){

                    String message = "No cdm object was found for uuid " + old.getUuid() + " of class " + old.getClass().getSimpleName();

                    importResult.addWarning(message);

                }else{

                    newMap.put(key, CdmBase.deproxy(cdmBase));

                }

            }else{

                String message = "Value for key " +  key + " was null in deduplication map";

                importResult.addWarning(message);

            }

        }

        return newMap;

    }

    private <T extends ICdmBase> Map<String, Set<T>> refreshSetMap(Map<String, Set<T>> oldMap,

            IService<T> service, ImportResult importResult) {

        Map<String, Set<T>> newMap = new HashMap<>();

        logger.debug("Start loading map");  //TODO debug only

        //create UUID set

        Set<UUID> uuidSet = new HashSet<>();

        for (String key : oldMap.keySet()){

            Set<T> oldSet = oldMap.get(key);

            for (T item : oldSet){

                UUID uuid = item.getUuid();

                uuidSet.add(uuid);

            }

        }

        //create uuid-item map

        Map<UUID, T> itemMap = new HashMap<>();

        List<T> list = service.find(uuidSet);

        for (T item : list){

            itemMap.put(item.getUuid(), item);

        }

        //refresh

        for (String key : oldMap.keySet()){

            Set<T> oldSet = oldMap.get(key);

            Set<T> newSet = new HashSet<>();

            if (oldSet != null){

                newMap.put(key, newSet);

                for (T item : oldSet){

                    T cdmBase = CdmBase.deproxy(itemMap.get(item.getUuid()));

                    if (cdmBase == null){

                        String message = "No cdm object was found for uuid " + item.getUuid() + " of class " + item.getClass().getSimpleName();

                        importResult.addWarning(message);

                    }else{

                        newSet.add(cdmBase);

                    }

                }

            }else{

                String message = "Value for key " +  key + " was null in deduplication map";

                importResult.addWarning(message);

            }

        }

        return newMap;

    }

//************************ PUTTER / GETTER *****************************/

    //ENTITY

    private <S extends IdentifiableEntity<?>> void putEntity(String title, S entity, Map<String,Set<S>> map){

        Set<S> entitySet = map.get(title);

        if (entitySet == null){

            entitySet = new HashSet<>();

            map.put(title, entitySet);

        }

        entitySet.add(CdmBase.deproxy(entity));

    }

    private <S extends IdentifiableEntity> Set<S> getEntityByTitle(String title, DedupInfo<S> dedupInfo){

        return dedupInfo.map.get(title);

    }

    private <S extends IdentifiableEntity> Optional<S> getMatchingEntity(S entityOrig, DedupInfo<S> dedupInfo){

        S entity = CdmBase.deproxy(entityOrig);

        Predicate<S> matchFilter = reference ->{

            try {

                return dedupInfo.matcher.invoke((IMatchable)reference, (IMatchable)entity).isSuccessful();

            } catch (MatchException e) {

                throw new RuntimeException(e);

            }

        };

        Optional<S> result = Optional.ofNullable(getEntityByTitle(entity.getTitleCache(), dedupInfo))

                .orElse(new HashSet<>())

                .stream()

                .filter(matchFilter)

                .findAny();

        if (result.isPresent() || dedupInfo.status == Status.USE_MAP  || repository == null){

            return result;

        }else {

            try {

                return (Optional<S>)repository.getCommonService().findMatching((IMatchable)entity, dedupInfo.matcher).stream().findFirst();

            } catch (MatchException e) {

                throw new RuntimeException(e);

            }

        }

    }

    // AGENTS

    private void putAgentBase(String title, AgentBase<?> agent){

        if (agent.isInstanceOf(Person.class) ){

            putEntity(title, CdmBase.deproxy(agent, Person.class), personDedupInfo.map);

        }else if (agent.isInstanceOf(Team.class)){

            putEntity(title, CdmBase.deproxy(agent, Team.class), teamDedupInfo.map);

        }else{

            putEntity(title, CdmBase.deproxy(agent, Institution.class), institutionMap);

        }

    }

    private <T extends TeamOrPersonBase<?>> T getTeamOrPerson(T agent){

        T result = agent;

        if (agent.isInstanceOf(Person.class)){

            result = (T)getMatchingEntity(CdmBase.deproxy(agent, Person.class), personDedupInfo).orElse(null) ; // personMap.get(title);

        }else if (agent.isInstanceOf(Team.class)) {

            result = (T)getMatchingEntity(CdmBase.deproxy(agent, Team.class), teamDedupInfo).orElse(null); // teamMap.get(title);

        }

        return result;

    }

    //COLLECTIONS

    private Set<Collection> getCollections(String title){

        return collectionMap.get(title);

    }

    private Optional<Collection> getMatchingCollections(Collection existing){

        Predicate<Collection> matchFilter = collection ->{

//            try {

                //TODO right Collection matching

                if (CdmUtils.nullSafeEqual(collection.getName(), existing.getName())

                        && CdmUtils.nullSafeEqual(collection.getCode(), existing.getCode())){

                    return true;

                }else{

                    return false;

                }

//                return collectionMatcher.invoke(collection, existing);

//            } catch (MatchException e) {

//                throw new RuntimeException(e);

//            }

        };

        return Optional.ofNullable(getCollections(existing.getTitleCache()))

                .orElse(new HashSet<>())

                .stream()

                .filter(matchFilter)

                .findAny();

    }

// **************************** METHODS *****************************/

    /**

     * This method replaces name authors, nomenclatural reference and

     * nomenclatural reference author by existing authors and references

     * if matching authors or references exist. If not, the given authors

     * and references are added to the map of existing entities.

     *

     * @param state the import state

     * @param name the name with authors and references to replace

     */

    public void replaceAuthorNamesAndNomRef(INonViralName name) {

        TeamOrPersonBase<?> combAuthor = name.getCombinationAuthorship();

        name.setCombinationAuthorship(getExistingAuthor(combAuthor));

        TeamOrPersonBase<?> exAuthor = name.getExCombinationAuthorship();

        name.setExCombinationAuthorship(getExistingAuthor(exAuthor));

        TeamOrPersonBase<?> basioAuthor = name.getBasionymAuthorship();

        name.setBasionymAuthorship(getExistingAuthor(basioAuthor));

        TeamOrPersonBase<?> exBasioAuthor = name.getExBasionymAuthorship();

        name.setExBasionymAuthorship(getExistingAuthor(exBasioAuthor));

        INomenclaturalReference nomRef = name.getNomenclaturalReference();

        if (nomRef != null){

            TeamOrPersonBase<?> refAuthor = nomRef.getAuthorship();

            nomRef.setAuthorship(getExistingAuthor(refAuthor));

            Reference existingRef = getExistingReference((Reference)nomRef);

            //TODO AM: why do we need to check null here (we don't do this for authors, maybe because it is an original source?)

            if (existingRef != null){

                name.setNomenclaturalReference(existingRef);

            }

        }

    }

    public void replaceReferenceRelatedData(Reference ref) {

        TeamOrPersonBase<?> author = ref.getAuthorship();

        ref.setAuthorship(getExistingAuthor(author));

        ref.setInReference(getExistingReference(ref.getInReference()));

    }

    public <T extends TeamOrPersonBase<?>> T getExistingAuthor(T author) {

        if (author == null){

            return null;

        }else{

            //TODO

            init(personDedupInfo);

            init(teamDedupInfo);

            initAuthorTitleCaches(author);

            T result = getTeamOrPerson(author);

            if (result == null){

                putAgentBase(author.getTitleCache(), author);

                if (author.isInstanceOf(Team.class)){

                    handleTeam(CdmBase.deproxy(author, Team.class));

                }

                result = author;

            }

            return result;

        }

    }

    private <T extends TeamOrPersonBase<?>> void initAuthorTitleCaches(T teamOrPerson) {

        //NOTE: this is more or less redundant copy from CdmPreDataChangeListener

        if (teamOrPerson.isInstanceOf(Team.class)){

            Team team = CdmBase.deproxy(teamOrPerson, Team.class);

            if (!team.isProtectedNomenclaturalTitleCache()){

                team.setNomenclaturalTitleCache(null, false);

            }

            if (!team.isProtectedCollectorTitleCache()){

                team.setCollectorTitleCache(null, false);

            }

        }

        teamOrPerson.getNomenclaturalTitleCache();

        teamOrPerson.getCollectorTitleCache();

        if (! teamOrPerson.isProtectedTitleCache()){

            teamOrPerson.setTitleCache(teamOrPerson.generateTitle(), false);

        }

    }

    private void initReferenceCaches(Reference ref) {

        ////TODO better do via matching strategy  (newReference might have caches == null)

        //more or less copy from CdmPreDataChangeListener

        ref.getAbbrevTitleCache();

        ref.getTitleCache();

   }

    public AgentBase<?> getExistingAgent(AgentBase<?> agent) {

        if (agent == null){

            return null;

        } else if (agent.isInstanceOf(TeamOrPersonBase.class)){

            return getExistingAuthor(CdmBase.deproxy(agent, TeamOrPersonBase.class));

        }else{

            throw new RuntimeException("Institution matching not yet implemented");

//            initInstitutionMap();

//            Set<Institution> result = institutionMap.get(agent.getTitleCache());

//            if (result == null){

//                result = putEntity(agent.getTitleCache(), CdmBase.deproxy(agent, Institution.class), institutionMap);

//            }

//            return result;

        }

    }

    private <S extends IdentifiableEntity<?>> void init(DedupInfo<S> dedupInfo) {

        dedupInfo.status = init(dedupInfo.clazz, dedupInfo.status, dedupInfo.map);

    }

    private <S extends IdentifiableEntity<?>> Status init(Class<S> clazz, Status status, Map<String,Set<S>> map) {

        if (status == Status.NOT_INIT && repository != null){

            if (maxCountFullLoad != NEVER_USE_MAP){

                long nExisting = -2;

                if (maxCountFullLoad != ALWAYS_USE_MAP){

                    nExisting = repository.getCommonService().count(clazz);

                }

                if (nExisting <= maxCountFullLoad ){

                    List<String> propertyPaths = Arrays.asList("");

                    List<S> existingEntities = repository.getCommonService().list(clazz, null, null, null, propertyPaths);

                    for (S ref : existingEntities){

                        putEntity(ref.getTitleCache(), ref, map);

                    }

                    return Status.USE_MAP;

                }else{

                    return Status.USE_REPO;

                }

            }else{

                return Status.USE_REPO;

            }

        }

        return status;

    }

    private void handleTeam(Team team) {

        List<Person> members = team.getTeamMembers();

        for (int i =0; i< members.size(); i++){

            Person person = CdmBase.deproxy(members.get(i));

            Person existingPerson = getMatchingEntity(person, personDedupInfo).orElse(null);

            if (existingPerson != null){

                members.set(i, existingPerson);

            }else{

                putAgentBase(person.getTitleCache(), person);

            }

        }

    }

    public Collection getExistingCollection(Collection collection) {

        if (collection == null){

            return null;

        }else{

            initCollectionMap();

            Collection result = getMatchingCollections(collection).orElse(null);

            if (result == null){

                result = collection;

                putEntity(result.getTitleCache(), result, collectionMap);

            }else{

                if(logger.isDebugEnabled()) {

                    logger.debug("Matches");

                 }

            }

            return result;

        }

    }

    private void initCollectionMap() {

        collectionStatus = init(Collection.class, collectionStatus, collectionMap);

    }

   public Reference getExistingReference(Reference ref) {

       if (ref == null){

           return null;

       }else{

           init(referenceDedupInfo);

           initReferenceCaches(ref);

           Reference result = getMatchingEntity(ref, referenceDedupInfo).orElse(null);

           if (result == null){

               result = ref;

               Reference inRef = result.getInReference();

               if (inRef != null){

                   result.setInReference(getExistingReference(result.getInReference()));

               }

               putEntity(result.getTitleCache(), result, referenceDedupInfo.map);

           }else{

               if(logger.isDebugEnabled()) {logger.debug("Matches");}

           }

           return result;

       }

   }

   public TaxonName getExistingName(TaxonName name) {

       if (name == null){

           return null;

       }else{

           init(nameDedupInfo);

           TaxonName result = getMatchingEntity(name, nameDedupInfo).orElse(null);

           if (result == null){

               result = name;

               Set<HybridRelationship> parentRelations = result.getHybridChildRelations();

               for (HybridRelationship rel : parentRelations){

                   TaxonName parent = rel.getParentName();

                   if (parent != null){

                       rel.setParentName(getExistingName(parent));

                   }

               }

               putEntity(result.getTitleCache(), result, nameDedupInfo.map);

           }else{

               if(logger.isDebugEnabled()) {

                   logger.debug("Matches");

                }

           }

           return result;

       }

   }

   public Rights getExistingCopyright(Rights right) {

       if (right == null || !RightsType.COPYRIGHT().equals(right.getType())){

           return null;

       }else{

           initCopyrightMap();

           String key = makeCopyrightKey(right);

           Set<Rights> set = copyrightMap.get(key);

           if (set == null || set.isEmpty()){

               putCopyright(key, right);

               return right;

           }else if (set.size()>1){

               //TODO

               logger.warn("More than 1 matching copyright not yet handled for key: " + key);

           }

           return set.iterator().next();

       }

   }

    private void initCopyrightMap() {

        if (copyrightStatus == Status.NOT_INIT && repository != null){

            List<String> propertyPaths = Arrays.asList("");

            List<Rights> existingRights = repository.getRightsService().list(null, null, null, null, propertyPaths);

            for (Rights right : existingRights){

                if (RightsType.COPYRIGHT().equals(right.getType())){

                    putCopyright(makeCopyrightKey(right), right);

                }

            }

            copyrightStatus = Status.USE_MAP;

        }

    }

    private void putCopyright(String key, Rights right) {

        Set<Rights> rights = copyrightMap.get(key);

        if (rights == null){

            rights = new HashSet<>();

            copyrightMap.put(key, rights);

        }

        rights.add(CdmBase.deproxy(right));

    }

    private String makeCopyrightKey(Rights right) {

        if (right.getAgent() != null){

            return right.getAgent().getTitleCache();

        }else if (right.getText() != null){

            return right.getText();

        }else {

            logger.warn("Key for copyright could not be created: " + right);

            return right.getUuid().toString();

        }

    }

}