EDIT

import eu.etaxonomy.cdm.model.common.IdentifiableEntity;

import eu.etaxonomy.cdm.strategy.match.IMatchable;

 * Note 2021: Was originally used as fast deduplication tool for commandline imports

 * into empty databases. Currently it is transformed into a deduplication tool that

 * can be used during application based imports.

 *

    //for possible future use

    @SuppressWarnings("unused")

    public static final int NEVER_USE_MAP = 0;

    public static final int ALWAYS_USE_MAP = -1;

    //should deduplication use maps indexing the full database content? If yes, what is the maximum number of records for this.

    //If more records exist deduplication is done on the fly.

    //0 = never use map

    //-1 = always use map

    private int maxCountFullLoad = ALWAYS_USE_MAP;

    public int getMaxCountFullLoad() {

        return maxCountFullLoad;

    }

    public void setMaxCountFullLoad(int maxCountFullLoad) {

        this.maxCountFullLoad = maxCountFullLoad;

    }

    private enum Status{

        NOT_INIT,

        USE_MAP,

        USE_REPO;

    }

    private class DedupInfo<S extends IdentifiableEntity>{

        Class<S> clazz;

        IMatchStrategyEqual matcher;

        Map<String, Set<S>> map = new HashMap<>();

        Status status = Status.NOT_INIT;

        @SuppressWarnings("unchecked")

        private DedupInfo(Class<S> clazz, DedupMap dedupMap){

            this.clazz = clazz;

            if (IMatchable.class.isAssignableFrom(clazz)) {

                matcher = DefaultMatchStrategy.NewInstance((Class<IMatchable>)clazz);

            }

            dedupMap.put(clazz, this);

        }

        @Override

        public String toString() {

            return clazz.getSimpleName() + ":" + status.name()+":mapsize=" + map.size()+":"+ (matcher == null?"without":"with") + " matcher";

        }

    }

    private class DedupMap<T extends IdentifiableEntity> extends HashMap<Class<T>, DedupInfo<T>>{

        private static final long serialVersionUID = 3757206594833330646L;

    }

    private DedupMap<? extends IdentifiableEntity> dedupMap = new DedupMap<>();

    private DedupInfo<Reference> referenceDedupInfo = new DedupInfo<>(Reference.class, dedupMap);

    private DedupInfo<Person> personDedupInfo = new DedupInfo<>(Person.class, dedupMap);

    private DedupInfo<Team> teamDedupInfo = new DedupInfo<>(Team.class, dedupMap);

    private DedupInfo<TaxonName> nameDedupInfo = new DedupInfo<>(TaxonName.class, dedupMap);

    private Status institutionStatus = Status.NOT_INIT;

    private Status copyrightStatus = Status.NOT_INIT;

    private Status collectionStatus = Status.NOT_INIT;

    private Map<String, Set<Institution>> institutionMap = new HashMap<>();

        dedupMap.values().forEach(di->di.map.clear());

             dedupMap.get(Reference.class).matcher.setMatchMode("title", MatchMode.EQUAL);

             dedupMap.get(Team.class).matcher.setMatchMode("nomenclaturalTitleCache", MatchMode.EQUAL_OR_SECOND_NULL);

    /**

     * Clears all internal maps and loads them with same data as before but in current session.

     */

        referenceDedupInfo.map = refreshSetMap(referenceDedupInfo.map, (IService)repository.getReferenceService(), importResult);

        personDedupInfo.map = refreshSetMap(personDedupInfo.map, (IService)repository.getAgentService(), importResult);

        teamDedupInfo.map = refreshSetMap(teamDedupInfo.map, (IService)repository.getAgentService(), importResult);

        institutionMap = refreshSetMap(institutionMap, (IService)repository.getAgentService(), importResult);

        nameDedupInfo.map = refreshSetMap(nameDedupInfo.map, (IService)repository.getNameService(), importResult);

    //maybe this was used for Institution before

    //ENTITY

    private <S extends IdentifiableEntity<?>> void putEntity(String title, S entity, Map<String,Set<S>> map){

        Set<S> entitySet = map.get(title);

        if (entitySet == null){

            entitySet = new HashSet<>();

            map.put(title, entitySet);

        entitySet.add(CdmBase.deproxy(entity));

    private <S extends IdentifiableEntity> Set<S> getEntityByTitle(String title, DedupInfo<S> dedupInfo){

        return dedupInfo.map.get(title);

    private <S extends IdentifiableEntity> Optional<S> getMatchingEntity(S entityOrig, DedupInfo<S> dedupInfo){

        S entity = CdmBase.deproxy(entityOrig);

        Predicate<S> matchFilter = reference ->{

                return dedupInfo.matcher.invoke((IMatchable)reference, (IMatchable)entity).isSuccessful();

        Optional<S> result = Optional.ofNullable(getEntityByTitle(entity.getTitleCache(), dedupInfo))

        if (result.isPresent() || dedupInfo.status == Status.USE_MAP  || repository == null){

            return result;

        }else {

            try {

                return (Optional<S>)repository.getCommonService().findMatching((IMatchable)entity, dedupInfo.matcher).stream().findFirst();

            } catch (MatchException e) {

                throw new RuntimeException(e);

            }

        }

            putEntity(title, CdmBase.deproxy(agent, Person.class), personDedupInfo.map);

            putEntity(title, CdmBase.deproxy(agent, Team.class), teamDedupInfo.map);

            putEntity(title, CdmBase.deproxy(agent, Institution.class), institutionMap);

            result = (T)getMatchingEntity(CdmBase.deproxy(agent, Person.class), personDedupInfo).orElse(null) ; // personMap.get(title);

            result = (T)getMatchingEntity(CdmBase.deproxy(agent, Team.class), teamDedupInfo).orElse(null); // teamMap.get(title);

            //TODO AM: why do we need to check null here (we don't do this for authors, maybe because it is an original source?)

    public void replaceReferenceRelatedData(Reference ref) {

        TeamOrPersonBase<?> author = ref.getAuthorship();

        ref.setAuthorship(getExistingAuthor(author));

        ref.setInReference(getExistingReference(ref.getInReference()));

    }

            //TODO

            init(personDedupInfo);

            init(teamDedupInfo);

            throw new RuntimeException("Institution matching not yet implemented");

//            initInstitutionMap();

//            Set<Institution> result = institutionMap.get(agent.getTitleCache());

//            if (result == null){

//                result = putEntity(agent.getTitleCache(), CdmBase.deproxy(agent, Institution.class), institutionMap);

//            }

//            return result;

    private <S extends IdentifiableEntity<?>> void init(DedupInfo<S> dedupInfo) {

        dedupInfo.status = init(dedupInfo.clazz, dedupInfo.status, dedupInfo.map);

    }

    private <S extends IdentifiableEntity<?>> Status init(Class<S> clazz, Status status, Map<String,Set<S>> map) {

        if (status == Status.NOT_INIT && repository != null){

            if (maxCountFullLoad != NEVER_USE_MAP){

                long nExisting = -2;

                if (maxCountFullLoad != ALWAYS_USE_MAP){

                    nExisting = repository.getCommonService().count(clazz);

                }

                if (nExisting <= maxCountFullLoad ){

                    List<String> propertyPaths = Arrays.asList("");

                    List<S> existingEntities = repository.getCommonService().list(clazz, null, null, null, propertyPaths);

                    for (S ref : existingEntities){

                        putEntity(ref.getTitleCache(), ref, map);

                    }

                    return Status.USE_MAP;

                }else{

                    return Status.USE_REPO;

                }

            }else{

                return Status.USE_REPO;

        return status;

            Person existingPerson = getMatchingEntity(person, personDedupInfo).orElse(null);

                putEntity(result.getTitleCache(), result, collectionMap);

        collectionStatus = init(Collection.class, collectionStatus, collectionMap);

           init(referenceDedupInfo);

           Reference result = getMatchingEntity(ref, referenceDedupInfo).orElse(null);

               putEntity(result.getTitleCache(), result, referenceDedupInfo.map);

               if(logger.isDebugEnabled()) {logger.debug("Matches");}

   public TaxonName getExistingName(TaxonName name) {

           init(nameDedupInfo);

           TaxonName result = getMatchingEntity(name, nameDedupInfo).orElse(null);

                   TaxonName parent = rel.getParentName();

               putEntity(result.getTitleCache(), result, nameDedupInfo.map);

        if (copyrightStatus == Status.NOT_INIT && repository != null){

            copyrightStatus = Status.USE_MAP;

import eu.etaxonomy.cdm.io.common.utils.ImportDeduplicationHelper;

                    Reference existingRef = state.getDeduplicationHelper().getExistingReference(ref);

                    if (existingRef == ref){ //reference does not yet exist so the identical reference has been returned

                        state.getDeduplicationHelper().replaceReferenceRelatedData(ref);

                        referencesToSave.add(ref);

                        if (ref.getInReference() != null){

                            referencesToSave.add(ref.getInReference());

                        }

                    }else{

                        //merge ?

    @Override

    public ImportDeduplicationHelper createDeduplicationHelper(RisReferenceImportState state){

        ImportDeduplicationHelper result = super.createDeduplicationHelper(state);

        result.setMaxCountFullLoad(state.getConfig().getDeduplicationMaxCountForFullLoad());

        return result;

    }

    private int deduplicationMaxCountForFullLoad = 200;

//********************************** FACTORY ***********************************/

        RisReferenceImportConfigurator result = new RisReferenceImportConfigurator(null, cdm);

//************************ CONSTRUCTOR ****************************************/

    private RisReferenceImportConfigurator(URI uri, ICdmDataSource cdm) {

// ************************* GETTER / SETTER *************************************/

    public int getDeduplicationMaxCountForFullLoad() {

        return deduplicationMaxCountForFullLoad;

    }

    public void setDeduplicationMaxCountForFullLoad(int deduplicationMaxCountForFullLoad) {

        this.deduplicationMaxCountForFullLoad = deduplicationMaxCountForFullLoad;

    }

//********************** METHODS ********************************************/

    @SuppressWarnings("unchecked")

}

import java.util.Collections;

import eu.etaxonomy.cdm.api.service.IAgentService;

import eu.etaxonomy.cdm.model.agent.Institution;

import eu.etaxonomy.cdm.model.common.IdentifiableEntity;

	@SpringBeanByType

    private IAgentService agentService;

	public void setUp() {}

	    RisReferenceImportConfigurator configurator = getConfigurator("RisReferenceImportTest-input.ris");

	public void testChapter() {

	    final RisReferenceImportConfigurator configurator = getConfigurator("Arias2012.ris");

        List<Reference> referenceList = referenceService.list(Reference.class, null, null, null, null);

                3, referenceList.size());

        Reference bookSection = referenceList.stream().filter(r->r.getType() == ReferenceType.BookSection).findFirst().get();

        Reference book = referenceList.stream().filter(r->r.getType() == ReferenceType.Book).findFirst().get();

        Reference sourceRef = referenceList.stream().filter(r->r.equals(configurator.getSourceReference())).findFirst().get();

        List<Person> personList = agentService.list(Person.class, null, null, null, null);

        Assert.assertEquals("There should be 5 persons", 5, personList.size());

        List<Team> teamList = agentService.list(Team.class, null, null, null, null);

        Assert.assertEquals("There should be 1 team", 1, teamList.size());

        //test deduplication by running it again

        result = defaultImport.invoke(configurator);

        report = result.createReport().toString();

        Assert.assertTrue(report.contains("Reference: 0"));

        Assert.assertEquals(0, result.getErrors().size() + result.getExceptions().size() + result.getWarnings().size());

        referenceList = referenceService.list(Reference.class, null, null, null, null);

        Assert.assertEquals("There should still be 3 references, the book-section, the book and the source reference",

                3, referenceList.size());

        personList = agentService.list(Person.class, null, null, null, null);

        Assert.assertEquals("There should still be 5 persons", 5, personList.size());

        teamList = agentService.list(Team.class, null, null, null, null);

        Assert.assertEquals("There should still be 1 team", 1, teamList.size());

        //test deduplication by running another chapter

        RisReferenceImportConfigurator configurator2 = getConfigurator("Arias2012_2.ris");

        result = defaultImport.invoke(configurator2);

        report = result.createReport().toString();

//        Assert.assertTrue(report.contains("Reference: 0"));

        Assert.assertEquals(0, result.getErrors().size() + result.getExceptions().size() + result.getWarnings().size());

        referenceList = referenceService.list(Reference.class, null, null, null, null);

        Assert.assertEquals("There should be 5 references, 2 book-sections, the book and 2 source references",

                5, referenceList.size());

        personList = agentService.list(Person.class, null, null, null, null);

        Assert.assertEquals("There should be 6 persons now", 6, personList.size());

        teamList = agentService.list(Team.class, null, null, null, null);

        Assert.assertEquals("There should be 2 teams now", 2, teamList.size());

    private RisReferenceImportConfigurator getConfigurator(String fileName) {

        String inputFile = "/eu/etaxonomy/cdm/io/reference/ris/in/" + fileName;

        URL url = this.getClass().getResource(inputFile);

        assertNotNull("URL for the test file '" + inputFile + "' does not exist", url);

        try {

            RisReferenceImportConfigurator result = RisReferenceImportConfigurator.NewInstance(url, null);

            result.setDeduplicationMaxCountForFullLoad(1);

            return result;

        } catch (IOException e) {

            Assert.fail("IOException while creating configurator: " + e.getMessage());

            return null;

        }

    }

    @Test

    public void testLongFile() {

        RisReferenceImportConfigurator configurator = getConfigurator("Acantholimon.ris");

        ImportResult result = defaultImport.invoke(configurator);

        @SuppressWarnings("unused")

        String report = result.createReport().toString();

//        System.out.println(report);

//        Integer expectedWithoutDeduplication = 118;  //did not count yet

        Integer expectedDeduplicated = 104;  //did not count yet

        Assert.assertEquals(expectedDeduplicated, result.getNewRecords(Reference.class));

//        System.out.println("Person: "+ result.getNewRecords(Person.class));

//        System.out.println("Team: "+ result.getNewRecords(Team.class));

        List<Reference> refList = referenceService.list(Reference.class, null, null, null, null);

//        Assert.assertEquals("There should be 119 references (still need to count them)", 119, refList.size());

        Collections.sort(refList, (r1,r2) -> r1.getTitleCache().compareTo(r2.getTitleCache()));

        printList(refList);

        List<Person> personList = agentService.list(Person.class, null, null, null, null);

        printList(personList);

        Assert.assertEquals(99, personList.size());

        List<Team> teamList = agentService.list(Team.class, null, null, null, null);

        printList(teamList);

        Assert.assertEquals(33, teamList.size());

        List<Institution> institutionList = agentService.list(Institution.class, null, null, null, null);

        printList(institutionList);

        Assert.assertEquals(0, institutionList.size());

        Reference ref58 = refList.stream().filter(r->hasId(r, "58", false)).findFirst().get();

        Reference ref53 = refList.stream().filter(r->hasId(r, "53", false)).findFirst().get();

    private void printList(List<? extends IdentifiableEntity<?>> list) {

        if (!logger.isDebugEnabled()){

            return;

        }

        System.out.println(list.size());

        Collections.sort(list, (p1,p2) -> p1.getTitleCache().compareTo(p2.getTitleCache()));

        list.stream().forEach(r->System.out.println(r.getTitleCache()));

    }

TY  - CHAP

A2  - Medina, L. R.

AU  - Arias, S.

AU  - Gama-Cruz, A.

CY  - México D. F.

PB  - Instituto de Biología, Universidad Nacional Autónoma de México

PY  - 2012

SP  - 236-247

ST  - Cactaceae2

T2  - Flora del Valle de Tehuacán-Cuicatlán

TI  - Cactaceae2

VL  - Fascículo 95

ID  - 3825

ER  -