Project

General

Profile

« Previous | Next » 

Revision da6c2341

Added by Andreas Müller almost 5 years ago

ref #7823 deduplicate usernames depending on secrefs, use xml imports (and other imports) as source, not username

View differences:

app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelImportBase.java
17 17
import org.apache.log4j.Logger;
18 18
import org.joda.time.DateTime;
19 19

  
20
import eu.etaxonomy.cdm.api.service.pager.Pager;
20 21
import eu.etaxonomy.cdm.common.CdmUtils;
21 22
import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
22 23
import eu.etaxonomy.cdm.io.common.DbImportBase;
......
27 28
import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
28 29
import eu.etaxonomy.cdm.model.common.Annotation;
29 30
import eu.etaxonomy.cdm.model.common.AnnotationType;
31
import eu.etaxonomy.cdm.model.common.CdmBase;
30 32
import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
31 33
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
32 34
import eu.etaxonomy.cdm.model.common.Language;
33 35
import eu.etaxonomy.cdm.model.common.MarkerType;
36
import eu.etaxonomy.cdm.model.common.SourcedEntityBase;
34 37
import eu.etaxonomy.cdm.model.common.User;
35 38
import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
36 39
import eu.etaxonomy.cdm.model.location.Country;
......
38 41
import eu.etaxonomy.cdm.model.location.NamedAreaType;
39 42
import eu.etaxonomy.cdm.model.reference.ISourceable;
40 43
import eu.etaxonomy.cdm.model.reference.Reference;
44
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
41 45
import eu.etaxonomy.cdm.model.taxon.Taxon;
42 46
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
43 47
import eu.etaxonomy.cdm.model.term.TermType;
44 48
import eu.etaxonomy.cdm.model.term.TermVocabulary;
49
import eu.etaxonomy.cdm.persistence.query.MatchMode;
45 50

  
46 51
/**
47 52
 * @author a.mueller
......
107 112
		BerlinModelImportConfigurator config = state.getConfig();
108 113
		Object createdWhen = rs.getObject("Created_When");
109 114
		String createdWho = rs.getString("Created_Who");
110
		createdWho = handleHieraciumPilosella(createdWho);
115
		createdWho = normalizeUsername(state, createdWho);
111 116
		Object updatedWhen = null;
112 117
		String updatedWho = null;
113 118
		if (excludeUpdated == false){
114 119
			try {
115 120
				updatedWhen = rs.getObject("Updated_When");
116 121
				updatedWho = rs.getString("Updated_who");
122
				updatedWho = normalizeUsername(state, updatedWho);
117 123
			} catch (SQLException e) {
118 124
				//Table "Name" has no updated when/who
119 125
			}
120 126
		}
121
		String notes = rs.getString("notes");
122 127

  
123 128
		boolean success  = true;
124 129

  
......
136 141
			annotation.setAnnotationType(AnnotationType.TECHNICAL());
137 142
			annotatableEntity.addAnnotation(annotation);
138 143
		}else if (config.getEditor().equals(EDITOR.EDITOR_AS_EDITOR)){
139
			User creator = getUser(state, createdWho);
140
			User updator = getUser(state, updatedWho);
144
		    User creator;
145
		    boolean xmlSourceAdded= addXmlSource(state, annotatableEntity, createdWho, false);
146
		    if (xmlSourceAdded){
147
		        creator = getXmlImporter(state);
148
		    }else{
149
		        creator = getUser(state, createdWho);
150
		    }
151
		    annotatableEntity.setCreatedBy(creator);
152

  
153
		    User updator;
154
		    xmlSourceAdded = addXmlSource(state, annotatableEntity, updatedWho, xmlSourceAdded);
155
		    if (xmlSourceAdded){
156
		        updator = getXmlImporter(state);
157
		    }else{
158
		        updator = getUser(state, updatedWho);
159
		    }
160
			annotatableEntity.setUpdatedBy(updator);
161

  
141 162
			DateTime created = getDateTime(createdWhen);
142 163
			DateTime updated = getDateTime(updatedWhen);
143
			annotatableEntity.setCreatedBy(creator);
144
			annotatableEntity.setUpdatedBy(updator);
145 164
			annotatableEntity.setCreated(created);
146 165
			annotatableEntity.setUpdated(updated);
147 166
		}else {
......
151 170

  
152 171
		//notes
153 172
		if (! excludeNotes){
173
		    String notes = rs.getString("notes");
154 174
			doNotes(annotatableEntity, notes);
155 175
		}
156 176
		return success;
157 177
	}
158 178

  
159 179
	/**
160
	 * Special usecase for EDITWP6 import where in the createdWho field the original ID is stored
161
	 * @param createdWho
162
	 * @return
163
	 */
164
	private String handleHieraciumPilosella(String createdWho) {
165
		String result = createdWho;
166
		if (result == null){
167
			return null;
168
		}else if (result.startsWith("Hieracium_Pilosella import from EM")){
169
			return "Hieracium_Pilosella import from EM";
170
		}else{
171
			return result;
172
		}
173
	}
180
     * @param state
181
     * @return
182
     */
183
    private User getXmlImporter(BerlinModelImportState state) {
184
        return getUser(state, "import to BM");
185
    }
174 186

  
175
	private DateTime getDateTime(Object timeString){
187

  
188
    /**
189
     * @param state
190
	 * @param annotatableEntity
191
	 * @param xmlSourceAdded
192
     * @return
193
     */
194
    private boolean addXmlSource(BerlinModelImportState state, AnnotatableEntity annotatableEntity, String username, boolean existsAlready) {
195
        if (!state.getConfig().isEuroMed() || !isXmlImport(username)){
196
            return false;
197
        }
198
        if (isXmlImport(username) && existsAlready){
199
            return true;
200
        }
201
        Reference ref = getXmlRef(state, username);
202
        if (annotatableEntity.isInstanceOf(SourcedEntityBase.class)){
203
            SourcedEntityBase<?> sourcedEntity = CdmBase.deproxy(annotatableEntity, SourcedEntityBase.class);
204
            sourcedEntity.addImportSource(null, null, ref, null);
205
        }else if (annotatableEntity.isInstanceOf(DescriptionElementBase.class)){
206
            DescriptionElementBase descriptionElement = CdmBase.deproxy(annotatableEntity, DescriptionElementBase.class);
207
            descriptionElement.addImportSource(null, null, ref, null);
208
        }else{
209
            return false;
210
        }
211
        return true;
212
    }
213

  
214

  
215
    /**
216
     * @param state
217
     * @param username
218
     * @return
219
     */
220
    private Reference getXmlRef(BerlinModelImportState state, String username) {
221
        String namespace = "IMPORT USER";
222
        Reference ref = state.getRelatedObject(namespace, username, Reference.class);
223
        if (ref == null){
224
            if (state.getXmlImportRefUuid(username)!= null){
225
                ref = getReferenceService().find(state.getXmlImportRefUuid(username));
226
            }
227
            if (ref == null){
228
                Pager<Reference> pager = getReferenceService().findByTitle(Reference.class, username, MatchMode.EXACT, null, null, null, null, null);
229
                if (pager.getCount()>0){
230
                    ref = pager.getRecords().get(0);
231
                    if (pager.getCount()>1){
232
                        logger.warn("More then 1 reference found for " +  username);
233
                    }
234
                }
235
            }
236
            if (ref == null){
237
                ref = ReferenceFactory.newDatabase();
238
                ref.setTitleCache(username, true);
239
                ref.setTitle(username);
240
                getReferenceService().save(ref);
241
            }
242
            state.addRelatedObject(namespace, username, ref);
243
            state.putXmlImportRefUuid(username, ref.getUuid());
244

  
245
        }
246
        return ref;
247
    }
248

  
249

  
250
    /**
251
     * @param username
252
     * @return
253
     */
254
    private boolean isXmlImport(String username) {
255
        if (username == null){
256
            return false;
257
        }
258
        return username.matches(".*\\.xml")
259
                || username.equals("J.Li: import from pandora")
260
                || username.equals("J.Li imported from pandora")
261
                ||username.equals("Import from Kew Checklist 2010")
262
                ||username.equals("Import from ILDIS 2010");
263
    }
264

  
265

  
266
    private DateTime getDateTime(Object timeString){
176 267
		if (timeString == null){
177 268
			return null;
178 269
		}
......
420 511
        return false;
421 512
    }
422 513

  
514
    /**
515
     * @param state
516
     * @param username
517
     * @return
518
     */
519
    protected String normalizeUsername(BerlinModelImportState state, String username) {
520
        if (username == null){
521
            return null;
522
        }else{
523
            username = username.trim();
524
            if (state.getConfig().isEuroMed()){
525
                if (username.matches("[A-Za-z]+[7-9][0-9]")){
526
                    username = username.substring(0, username.length()-2);
527
                }else if(username.matches("(mariam[1-4]|palermo|palma|paltar)")){
528
                    username = "mariam";
529
                }
530
                if(username.matches("kapet")){
531
                    username = "kpet";
532
                }
533
            }
534
            return username;
535
        }
536
    }
537

  
423 538
}
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelImportState.java
1 1
/**
2 2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
3
* European Distributed Institute of Taxonomy
4 4
* http://www.e-taxonomy.eu
5
* 
5
*
6 6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7 7
* See LICENSE.TXT at the top of this package for the full license terms.
8 8
*/
......
11 11

  
12 12
import java.util.HashMap;
13 13
import java.util.Map;
14
import java.util.UUID;
14 15

  
15 16
import org.apache.log4j.Logger;
16 17

  
......
21 22
/**
22 23
 * @author a.mueller
23 24
 * @since 11.05.2009
24
 * @version 1.0
25 25
 */
26 26
public class BerlinModelImportState extends DbImportStateBase<BerlinModelImportConfigurator, BerlinModelImportState>{
27 27
	@SuppressWarnings("unused")
28 28
	private static final Logger logger = Logger.getLogger(BerlinModelImportState.class);
29 29

  
30
	private Map<String, DefinedTermBase> dbCdmDefTermMap = new HashMap<String, DefinedTermBase>();
31
	
30
	private Map<String, DefinedTermBase> dbCdmDefTermMap = new HashMap<>();
31

  
32 32
	private boolean isReferenceSecondPath = false;
33 33

  
34
	
35
	/* (non-Javadoc)
36
	 * @see eu.etaxonomy.cdm.io.common.IoStateBase#initialize(eu.etaxonomy.cdm.io.common.IoConfiguratorBase)
37
	 */
34
	private Map<String, UUID> xmlImportRefUuids = new HashMap<>();
35

  
36

  
38 37
	@Override
39 38
	public void initialize(BerlinModelImportConfigurator config) {
40 39
//		super(config);
......
50 49
	public Map<String, DefinedTermBase> getDbCdmDefinedTermMap(){
51 50
		return this.dbCdmDefTermMap;
52 51
	}
53
	
52

  
54 53
	public void putDefinedTermToMap(String tableName, String id, DefinedTermBase term){
55 54
		 this.dbCdmDefTermMap.put(tableName + "_" + id, term);
56 55
	}
57
	
56

  
58 57
	public void putDefinedTermToMap(String tableName, int id, DefinedTermBase term){
59 58
		putDefinedTermToMap(tableName, String.valueOf(id), term);
60 59
	}
......
69 68
	}
70 69

  
71 70

  
71
    public UUID getXmlImportRefUuid(String username) {
72
        return xmlImportRefUuids.get(username);
73
    }
74

  
75
    /**
76
     * @param username
77
     * @param uuid
78
     */
79
    public void putXmlImportRefUuid(String username, UUID uuid) {
80
        xmlImportRefUuids.put(username, uuid);
81
    }
82

  
83

  
84

  
72 85

  
73
    
74 86
}
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelUserImport.java
12 12
import java.sql.SQLException;
13 13
import java.util.ArrayList;
14 14
import java.util.Collection;
15
import java.util.HashSet;
15 16
import java.util.Map;
17
import java.util.Set;
16 18

  
17 19
import org.apache.log4j.Logger;
18 20
import org.springframework.stereotype.Component;
......
77 79
                " FROM " + dbTableName + " " ;
78 80
		ResultSet rs = source.getResultSet(strQuery) ;
79 81
		Collection<User> users = new ArrayList<>();
82
		Set<String> existingUsernames = new HashSet<>();
80 83

  
81 84

  
82 85
		TransactionStatus tx = this.startTransaction();
......
93 96
					Integer id = nullSafeInt(rs, "AuthorisationId");
94 97

  
95 98
					if (username != null){
96
						username = username.trim();
99
					    username = normalizeUsername(state, username);
100
					}
101
					if (existingUsernames.contains(username)){
102
					    continue;
97 103
					}
98 104
					User user = User.NewInstance(username, pwd);
99 105

  
......
122 128
					 */
123 129
					authenticate(Configuration.adminLogin, Configuration.adminPassword);
124 130
					getUserService().createUser(user);
131
					existingUsernames.add(username);
125 132

  
126 133
					users.add(user);
127 134
					state.putUser(username, user);
......
150 157
		return;
151 158
	}
152 159

  
153
	private Person deduplicatePerson(BerlinModelImportState state, Person person) {
160

  
161
    private Person deduplicatePerson(BerlinModelImportState state, Person person) {
154 162
        Person result = deduplicationHelper.getExistingAuthor(state, person);
155 163
        return result;
156 164
    }

Also available in: Unified diff