Project

General

Profile

« Previous | Next » 

Revision aae87424

Added by Andreas Müller over 1 year ago

ref #9771 improvements to Cdm2CdmImportBase including a first (not ready) version for VocabularyFilter

View differences:

cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/cdm2cdm/Cdm2CdmImportBase.java
714 714
    protected User handlePersistedUser(User user, Cdm2CdmImportState state) throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
715 715
        User result = (User)handlePersistedCdmBase(user, state);
716 716
        if (result.getUsername().equals("admin")){
717
            //TODO why only admin, is this not a problem for all duplicated usernames? Was this a preliminary decision?
717 718
            result = getUserService().listByUsername("admin", MatchMode.EXACT, null, null, null, null, null).iterator().next();
718 719
            state.putPermanent(user.getUuid(), result);
719 720
            cache(result, state); //necessary?
......
776 777
    protected <T extends CdmBase> T handlePersistedCdmBase(CdmBase cdmBase, Cdm2CdmImportState state) throws IllegalAccessException, InvocationTargetException, NoSuchFieldException, SecurityException, IllegalArgumentException, NoSuchMethodException {
777 778
        T result = (T)getTarget(cdmBase, state);
778 779
        //complete
779
        cdmBase.setCreatedBy(detach(cdmBase.getCreatedBy(), state));
780
        cdmBase.setCreatedBy(makeCreatedUpdatedBy(cdmBase.getCreatedBy(), state, false));
780 781
        return result;
781 782
    }
782 783

  
783 784
    protected <T extends VersionableEntity> T handlePersisted(VersionableEntity entity, Cdm2CdmImportState state) throws IllegalAccessException, InvocationTargetException, NoSuchFieldException, SecurityException, IllegalArgumentException, NoSuchMethodException {
784 785
        T result = (T)handlePersistedCdmBase((CdmBase)entity, state);
785 786
        //complete
786
        entity.setUpdatedBy(detach(entity.getUpdatedBy(), state));
787
        entity.setUpdatedBy(makeCreatedUpdatedBy(entity.getUpdatedBy(), state, true));
787 788
        return result;
788 789
    }
789 790

  
......
1020 1021
        return result;
1021 1022
    }
1022 1023

  
1024
// ****************************** USER HANDLING
1025

  
1026
    private User makeCreatedUpdatedBy(User createdByOriginal, Cdm2CdmImportState state, boolean isUpdatedBy) throws IllegalAccessException, InvocationTargetException, NoSuchFieldException, SecurityException, IllegalArgumentException, NoSuchMethodException {
1027
        UserImportMode mode = isUpdatedBy? state.getConfig().getUpdatedByMode() : state.getConfig().getCreatedByMode();
1028

  
1029
        switch (mode) {
1030
        case NONE:
1031
            return null;
1032
        case ORIGINAL:
1033
            return detach(createdByOriginal, state);
1034
        default:
1035
            logger.warn("Mode not yet supported: " + mode);
1036
            return null;
1037
        }
1038
    }
1039

  
1040

  
1023 1041
// ****************************** INVISIBLE **************************************/
1024 1042

  
1025 1043
    protected void setInvisible(Object holder, String fieldName, Object value) throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException {
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/cdm2cdm/Cdm2CdmImportConfigurator.java
15 15
import eu.etaxonomy.cdm.database.ICdmDataSource;
16 16
import eu.etaxonomy.cdm.database.ICdmImportSource;
17 17
import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
18
import eu.etaxonomy.cdm.filter.VocabularyFilter;
18 19
import eu.etaxonomy.cdm.io.common.ITaxonNodeOutStreamPartitioner;
19 20
import eu.etaxonomy.cdm.io.common.ImportConfiguratorBase;
20 21
import eu.etaxonomy.cdm.io.common.mapping.IInputTransformer;
......
35 36

  
36 37
    private TaxonNodeFilter taxonNodeFilter = TaxonNodeFilter.NewInstance();
37 38

  
38
    private Set<UUID> vocabularyFilter = new HashSet<>();
39
    private VocabularyFilter vocabularyFilter = VocabularyFilter.NewInstance();
39 40
    private Set<UUID> graphFilter = new HashSet<>();
40 41
    private ITaxonNodeOutStreamPartitioner partitioner;
41 42
    private boolean concurrent = false;  //
......
47 48
    private boolean addSources = true;
48 49
    private boolean removeImportSources = false;
49 50

  
51
    private UserImportMode createdByMode = UserImportMode.NONE;
52
    private UserImportMode updatedByMode = UserImportMode.NONE;
53

  
50 54
//***************************** NewInstance ************************/
51 55

  
52 56
    public static Cdm2CdmImportConfigurator NewInstace(ICdmImportSource source, ICdmDataSource destination){
......
144 148
        this.doVocabularies = doVocabularies;
145 149
    }
146 150

  
147
    public Set<UUID> getVocabularyFilter() {
151
    public VocabularyFilter getVocabularyFilter() {
148 152
        return vocabularyFilter;
149 153
    }
150
    public void setVocabularyFilter(Set<UUID> vocabularyFilter) {
154
    public void setVocabularyFilter(VocabularyFilter vocabularyFilter) {
151 155
        this.vocabularyFilter = vocabularyFilter;
152 156
    }
153 157

  
......
157 161
    public void setGraphFilter(Set<UUID> graphFilter) {
158 162
        this.graphFilter = graphFilter;
159 163
    }
164

  
165
    public UserImportMode getCreatedByMode() {
166
        return createdByMode;
167
    }
168
    public void setCreatedByMode(UserImportMode createdByMode) {
169
        this.createdByMode = createdByMode;
170
    }
171

  
172
    public UserImportMode getUpdatedByMode() {
173
        return updatedByMode;
174
    }
175
    public void setUpdatedByMode(UserImportMode updatedByMode) {
176
        this.updatedByMode = updatedByMode;
177
    }
160 178
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/cdm2cdm/Cdm2CdmVocabularyImport.java
19 19

  
20 20
import eu.etaxonomy.cdm.api.application.ICdmRepository;
21 21
import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
22
import eu.etaxonomy.cdm.filter.VocabularyFilter;
22 23
import eu.etaxonomy.cdm.model.term.DefinedTermBase;
23 24
import eu.etaxonomy.cdm.model.term.TermNode;
24 25
import eu.etaxonomy.cdm.model.term.TermTree;
......
53 54
        int count = 0;
54 55

  
55 56
        //vocabularies
56
        Set<UUID> vocUuids = state.getConfig().getVocabularyFilter();
57
        for (UUID vocUuid : vocUuids){
57
        VocabularyFilter vocFilter =
58
                state.getConfig().getVocabularyFilter();
59
        for (UUID vocUuid : getVocabularyService().uuidList(vocFilter)){
58 60
            TransactionStatus tx = startTransaction();
59 61
            doSingleVocabulary(state, vocUuid);
60 62
            commitTransaction(tx);
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/cdm2cdm/UserImportMode.java
1
// $Id$
2
/**
3
* Copyright (C) 2022 EDIT
4
* European Distributed Institute of Taxonomy
5
* http://www.e-taxonomy.eu
6
*
7
* The contents of this file are subject to the Mozilla Public License Version 1.1
8
* See LICENSE.TXT at the top of this package for the full license terms.
9
*/
10
package eu.etaxonomy.cdm.io.cdm2cdm;
11

  
12
/**
13
 * The mode how users should be handled during import.
14
 *
15
 * Note: This is not in the common package as CDM2CDM allows more modes (e.g. "original" then usual imports).
16
 *       But still should be discussed.
17
 *
18
 * @author a.mueller
19
 * @date 02.09.2022
20
 */
21
public enum UserImportMode {
22
    NONE,
23
    ORIGINAL,
24
//    CURRENT_USER,    not yet supported
25
//    ADMIN,
26
//    IMPORT_USER
27
    ;
28
}
cdmlib-io/src/test/java/eu/etaxonomy/cdm/io/cdm2cdm/Cdm2CdmVocabularyImportTest.java
32 32
import eu.etaxonomy.cdm.database.CdmDataSource;
33 33
import eu.etaxonomy.cdm.database.DbSchemaValidation;
34 34
import eu.etaxonomy.cdm.database.ICdmDataSource;
35
import eu.etaxonomy.cdm.filter.VocabularyFilter;
35 36
import eu.etaxonomy.cdm.io.common.CdmApplicationAwareDefaultImport;
36 37
import eu.etaxonomy.cdm.io.common.ImportResult;
37 38
import eu.etaxonomy.cdm.io.common.events.LoggingIoObserver;
......
129 130
        configurator = Cdm2CdmImportConfigurator.NewInstace(otherRepository, null);
130 131
        configurator.setDoTaxa(false);
131 132
        configurator.setDoDescriptions(false);
132
        configurator.setVocabularyFilter(new HashSet<>(Arrays.asList(uuidStructVoc)));
133
//        VocabularyFilter vocFilter = VocabularyFilter.NewTermTypeInstance(TermType.Structure);
134
        VocabularyFilter vocFilter = VocabularyFilter.NewInstance();
135
        vocFilter.orVocabulary(uuidStructVoc);
136
        configurator.setVocabularyFilter(vocFilter);
133 137

  
134 138
        configurator.addObserver(new LoggingIoObserver());
135 139
        assertNotNull("Configurator could not be created", configurator);
cdmlib-model/src/main/java/eu/etaxonomy/cdm/filter/VocabularyFilter.java
1
/**
2
* Copyright (C) 2022 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.filter;
10

  
11
import java.io.Serializable;
12
import java.util.ArrayList;
13
import java.util.Collections;
14
import java.util.List;
15
import java.util.UUID;
16

  
17
import eu.etaxonomy.cdm.filter.LogicFilter.Op;
18
import eu.etaxonomy.cdm.model.term.TermType;
19
import eu.etaxonomy.cdm.model.term.TermVocabulary;
20

  
21
/**
22
 * Filter used e.g. for additivity ontology import. Should work similar to TaxonNodeFilter.
23
 *
24
 * @author a.mueller
25
 * @date 02.09.2022
26
 */
27
public class VocabularyFilter implements Serializable {
28

  
29
    private static final long serialVersionUID = -8054720226785479796L;
30

  
31
    private List<TermType> termTypes = new ArrayList<>();
32
    private List<LogicFilter<TermVocabulary>> termVocabularies = new ArrayList<>();
33

  
34
    private ORDER orderBy = null;
35

  
36
    public enum ORDER{
37
        ID("voc.id"),
38
//        TREEINDEX("tn.treeIndex"),
39
//        TREEINDEX_DESC("tn.treeIndex DESC")
40
        ;
41
        String hql;
42
        private ORDER(String hql){
43
            this.hql = hql;
44
        }
45
        public String getHql(){
46
            return hql;
47
        }
48
    }
49

  
50
// *************************** FACTORY ********************************/
51

  
52
    public static VocabularyFilter NewInstance(){
53
        return new VocabularyFilter();
54
    }
55

  
56
    public static VocabularyFilter NewTermTypeInstance(TermType termType){
57
        return new VocabularyFilter().orTermType(termType);
58
    }
59

  
60
// ******************** CONSTRUCTOR ************************/
61

  
62
    private VocabularyFilter() {
63
        reset();
64
    }
65

  
66
// ******************** reset *****************************/
67

  
68
    public void reset(){
69
        resetTermTypes();
70
        resetTermVocabularies();
71
    }
72

  
73
    private void resetTermTypes() {
74
        termTypes = new ArrayList<>();
75
    }
76

  
77
    private void resetTermVocabularies() {
78
        termVocabularies = new ArrayList<>();
79
    }
80

  
81
// ******************** OR, XXX ****************************************/
82

  
83
    public VocabularyFilter orTermType(TermType termType){
84
//        termTypes.add( new LogicFilter<>(termType, Op.OR));
85
        termTypes.add(termType);
86
        return this;
87
    }
88

  
89
    public VocabularyFilter orVocabulary(UUID uuid){
90
        termVocabularies.add( new LogicFilter<>(TermVocabulary.class, uuid, Op.OR));
91
        return this;
92
    }
93

  
94

  
95
    public List<LogicFilter<TermVocabulary>>getTermVocabulariesFilter(){
96
        return Collections.unmodifiableList(termVocabularies);
97
    }
98

  
99
    public List<TermType>getTermTypesFilter(){
100
        return Collections.unmodifiableList(termTypes);
101
    }
102

  
103
    public ORDER getOrderBy() {
104
        return orderBy;
105
    }
106
    public void setOrder(ORDER orderBy) {
107
        this.orderBy = orderBy;
108
    }
109
}
cdmlib-persistence/src/main/java/eu/etaxonomy/cdm/persistence/dao/description/IVocabularyFilterDao.java
1
/**
2
* Copyright (C) 2022 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.persistence.dao.description;
10

  
11
import java.util.List;
12
import java.util.UUID;
13

  
14
import eu.etaxonomy.cdm.filter.VocabularyFilter;
15
import eu.etaxonomy.cdm.persistence.dao.taxon.ITaxonNodeFilterDao;
16

  
17
/**
18
 * TODO very similar to {@link ITaxonNodeFilterDao}. Maybe we can combine it.
19
 *
20
 * @author a.mueller
21
 * @date 02.09.2022
22
 */
23
public interface IVocabularyFilterDao {
24

  
25
    /**
26
     * Counts the number of vocabulary IDs returned
27
     * when calling {@link #listUuids(VocabularyFilter)}
28
     * @param filter the vocabulary filter
29
     * @return Count of vocabularies
30
     */
31
    public long count(VocabularyFilter filter);
32

  
33
    /**
34
     * Retrieve vocabulary {@link UUID uuids} defined by a
35
     * {@link VocabularyFilter vocabulary filter}.
36
     * @param filter the vocabulary filter
37
     * @return List of vocabulary {@link UUID uuids}
38
     */
39
    public List<UUID> listUuids(VocabularyFilter filter);
40

  
41

  
42
    /**
43
     * Retrieve vocabulary IDs defined by a
44
     * {@link VocabularyFilter vocabulary filter}.
45
     * @param filter the vocabulary filter
46
     * @return List of vocabulary IDs
47
     */
48
    public List<Integer> idList(VocabularyFilter filter);
49
}
cdmlib-persistence/src/main/java/eu/etaxonomy/cdm/persistence/dao/hibernate/description/VocabularyFilterDaoHibernateImpl.java
1
/**
2
* Copyright (C) 2022 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.persistence.dao.hibernate.description;
10

  
11
import java.util.ArrayList;
12
import java.util.List;
13
import java.util.UUID;
14

  
15
import org.hibernate.query.Query;
16
import org.springframework.stereotype.Repository;
17

  
18
import eu.etaxonomy.cdm.common.CdmUtils;
19
import eu.etaxonomy.cdm.filter.LogicFilter;
20
import eu.etaxonomy.cdm.filter.LogicFilter.Op;
21
import eu.etaxonomy.cdm.filter.VocabularyFilter;
22
import eu.etaxonomy.cdm.model.term.TermType;
23
import eu.etaxonomy.cdm.model.term.TermVocabulary;
24
import eu.etaxonomy.cdm.persistence.dao.description.IVocabularyFilterDao;
25
import eu.etaxonomy.cdm.persistence.dao.hibernate.common.CdmEntityDaoBase;
26

  
27
/**
28
 * @author a.mueller
29
 * @date 02.09.2022
30
 */
31
@Repository
32
public class VocabularyFilterDaoHibernateImpl
33
        extends CdmEntityDaoBase<TermVocabulary>
34
        implements IVocabularyFilterDao {
35

  
36
    public VocabularyFilterDaoHibernateImpl() {
37
        super(TermVocabulary.class);
38
    }
39

  
40
    @Override
41
    public List<Integer> idList(VocabularyFilter filter){
42
        String queryStr = query(filter, "tn.id");
43
        Query<Integer> query = getSession().createQuery(queryStr, Integer.class);
44
        List<Integer> list = query.list();
45
        list = deduplicate(list);
46
        return list;
47
    }
48

  
49
    @Override
50
    public long count(VocabularyFilter filter){
51
        String queryStr = query(filter, "count(*) as n ");
52
        Query<Long> query = getSession().createQuery(queryStr, Long.class);
53
        long result = query.uniqueResult();
54

  
55
        return result;
56
    }
57

  
58
    @Override
59
    public List<UUID> listUuids(VocabularyFilter filter){
60
        String queryStr = query(filter, "tn.uuid");
61
        Query<UUID> query = getSession().createQuery(queryStr, UUID.class);
62
        List<UUID> list = query.list();
63

  
64
        list = deduplicate(list);
65
        return list;
66
    }
67

  
68
    private <T> List<T> deduplicate(List<T> list) {
69
        List<T> result = new ArrayList<>();
70
        for (T uuid : list){
71
            if (!result.contains(uuid)){
72
                result.add(uuid);
73
            }
74
        }
75
        return result;
76
    }
77

  
78
    //maybe we will later want to have ordering included
79
    private String query(VocabularyFilter filter, String selectPart){
80
        String select = " SELECT " + selectPart;
81
        String from = getFrom(filter);
82
        String vocabularyFilter = getVocabularyFilter(filter);
83
        String termTypeFilter = getTermTypeFilter(filter);
84

  
85
        String fullFilter = getFullFilter(vocabularyFilter, termTypeFilter);
86
//        String groupBy = " GROUP BY tn.uuid ";
87
        String groupBy = "";
88
        String orderBy = getOrderBy(filter, selectPart);
89
        String fullQuery = select + from + " WHERE " + fullFilter + groupBy + orderBy;
90

  
91
        return fullQuery;
92
    }
93

  
94
    private String getFullFilter(String vocabularyFilter, String termTypeFilter) {
95
        String result = " (1=1 ";
96
        result = CdmUtils.concat(") AND (", result, vocabularyFilter, termTypeFilter) + ") ";
97
        return result;
98
    }
99

  
100
    private String getFrom(@SuppressWarnings("unused") VocabularyFilter filter){
101
        String from = " FROM TermVocabulary voc ";
102
        return from;
103
    }
104

  
105
    private String getOrderBy(VocabularyFilter filter, String selectPart) {
106
        String orderBy = "";
107
        if (filter.getOrderBy()!= null && !selectPart.contains("count")){
108
            orderBy = "ORDER BY " + filter.getOrderBy().getHql();
109
        }
110
        return orderBy;
111
    }
112

  
113
    private String getVocabularyFilter(VocabularyFilter filter) {
114
        String result = "";
115
        List<LogicFilter<TermVocabulary>> termVocabularyFilter = filter.getTermVocabulariesFilter();
116
        boolean isFirst = true;
117
        for (LogicFilter<TermVocabulary> singleFilter : termVocabularyFilter){
118
            String uuid = singleFilter.getUuid().toString();
119
            String op = isFirst ? "" : op2Hql(singleFilter.getOperator());
120
            result = String.format("(%s%s(tn.uuid = '%s'))", result, op, uuid);
121
            isFirst = false;
122
        }
123
        return result;
124
    }
125

  
126
    private String getTermTypeFilter(VocabularyFilter filter) {
127
        String result = "";
128
        List<TermType> termTypeFilter = filter.getTermTypesFilter();
129
        boolean isFirst = true;
130
        for (TermType termType : termTypeFilter){
131
            String strTermType = termType.getKey();
132
            String op = isFirst ? "" : op2Hql(Op.OR);
133
            result = String.format("(%s%s(voc.termType = '%s'))", result, op, strTermType);
134
            isFirst = false;
135
        }
136
        return result;
137
    }
138

  
139
    /**
140
     * Returns the HQL string for this operation
141
     */
142
    private String op2Hql(Op op){
143
        return op == Op.NOT ? " AND NOT " : op.toString();
144
    }
145

  
146
}
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/IVocabularyService.java
15 15
import java.util.UUID;
16 16

  
17 17
import eu.etaxonomy.cdm.api.service.pager.Pager;
18
import eu.etaxonomy.cdm.filter.VocabularyFilter;
18 19
import eu.etaxonomy.cdm.model.common.CdmClass;
19 20
import eu.etaxonomy.cdm.model.common.Language;
20 21
import eu.etaxonomy.cdm.model.term.DefinedTermBase;
......
197 198
     */
198 199
    public List<TermVocabularyDto> findFeatureVocabularyDtoByTermTypes(Set<CdmClass> availableFor);
199 200

  
201
    /**
202
     * Returns a list of vocabulary {@link UUID uuids} according to the given filter.
203
     */
204
    public List<UUID> uuidList(VocabularyFilter filter);
205

  
206
    /**
207
     * Returns a list of vocabulary IDs according to the given filter.
208
     */
209
    public List<Integer> idList(VocabularyFilter filter);
210

  
211
    /**
212
     * Returns the total count of vocabularies according to the given filter.
213
     */
214
    public long count(VocabularyFilter filter);
215

  
200 216
}
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/VocabularyServiceImpl.java
22 22
import eu.etaxonomy.cdm.api.service.pager.Pager;
23 23
import eu.etaxonomy.cdm.api.service.pager.impl.DefaultPagerImpl;
24 24
import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
25
import eu.etaxonomy.cdm.filter.VocabularyFilter;
25 26
import eu.etaxonomy.cdm.model.common.CdmClass;
26 27
import eu.etaxonomy.cdm.model.common.Language;
27 28
import eu.etaxonomy.cdm.model.term.DefinedTermBase;
28 29
import eu.etaxonomy.cdm.model.term.TermType;
29 30
import eu.etaxonomy.cdm.model.term.TermVocabulary;
31
import eu.etaxonomy.cdm.persistence.dao.description.IVocabularyFilterDao;
30 32
import eu.etaxonomy.cdm.persistence.dao.term.ITermVocabularyDao;
31 33
import eu.etaxonomy.cdm.persistence.dto.TermDto;
32 34
import eu.etaxonomy.cdm.persistence.dto.TermVocabularyDto;
......
41 43
    @Autowired
42 44
    private ITermService termService;
43 45

  
46
    @Autowired
47
    private IVocabularyFilterDao vocabularyFilterDao;
48

  
44 49
	@Override
45 50
    @Autowired
46 51
	protected void setDao(ITermVocabularyDao dao) {
......
201 206
    public List<TermVocabularyDto> findVocabularyDtoByVocabularyUuids(List<UUID> vocUuids) {
202 207
        return dao.findVocabularyDtoByUuids(vocUuids);
203 208
    }
209

  
210
    @Override
211
    public List<UUID> uuidList(VocabularyFilter filter){
212
        return vocabularyFilterDao.listUuids(filter);
213
    }
214

  
215
    @Override
216
    public List<Integer> idList(VocabularyFilter filter){
217
        return vocabularyFilterDao.idList(filter);
218
    }
219

  
220
    @Override
221
    public long count(VocabularyFilter filter) {
222
        return vocabularyFilterDao.count(filter);
223
    }
204 224
}

Also available in: Unified diff