Revision 48014773
Added by Andreas Müller almost 9 years ago
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportBase.java | ||
---|---|---|
1 | 1 |
/** |
2 | 2 |
* Copyright (C) 2007 EDIT |
3 |
* European Distributed Institute of Taxonomy
|
|
3 |
* European Distributed Institute of Taxonomy |
|
4 | 4 |
* http://www.e-taxonomy.eu |
5 |
*
|
|
5 |
* |
|
6 | 6 |
* The contents of this file are subject to the Mozilla Public License Version 1.1 |
7 | 7 |
* See LICENSE.TXT at the top of this package for the full license terms. |
8 | 8 |
*/ |
... | ... | |
55 | 55 |
*/ |
56 | 56 |
public abstract class GlobisImportBase<CDM_BASE extends CdmBase> extends CdmImportBase<GlobisImportConfigurator, GlobisImportState> implements ICdmIO<GlobisImportState>, IPartitionedIO<GlobisImportState> { |
57 | 57 |
private static final Logger logger = Logger.getLogger(GlobisImportBase.class); |
58 |
|
|
58 |
|
|
59 | 59 |
public static final UUID ID_IN_SOURCE_EXT_UUID = UUID.fromString("23dac094-e793-40a4-bad9-649fc4fcfd44"); |
60 |
|
|
60 |
|
|
61 | 61 |
//NAMESPACES |
62 |
|
|
62 |
|
|
63 | 63 |
protected static final String REFERENCE_NAMESPACE = "Literatur"; |
64 | 64 |
protected static final String TAXON_NAMESPACE = "current_species"; |
65 | 65 |
protected static final String COLLECTION_NAMESPACE = "Collection"; |
66 | 66 |
protected static final String IMAGE_NAMESPACE = "Einzelbilder"; |
67 | 67 |
protected static final String SPEC_TAX_NAMESPACE = "specTax"; |
68 | 68 |
protected static final String TYPE_NAMESPACE = "specTax.SpecTypeDepository"; |
69 |
|
|
70 |
private String pluralString; |
|
71 |
private String dbTableName; |
|
72 |
private Class cdmTargetClass; |
|
73 | 69 |
|
74 |
private INonViralNameParser<?> parser = NonViralNameParserImpl.NewInstance(); |
|
70 |
private final String pluralString; |
|
71 |
private final String dbTableName; |
|
72 |
private final Class cdmTargetClass; |
|
73 |
|
|
74 |
private final INonViralNameParser<?> parser = NonViralNameParserImpl.NewInstance(); |
|
75 |
|
|
75 | 76 |
|
76 |
|
|
77 | 77 |
/** |
78 | 78 |
* @param dbTableName |
79 |
* @param dbTableName2
|
|
79 |
* @param dbTableName2 |
|
80 | 80 |
*/ |
81 | 81 |
public GlobisImportBase(String pluralString, String dbTableName, Class<?> cdmTargetClass) { |
82 | 82 |
this.pluralString = pluralString; |
... | ... | |
84 | 84 |
this.cdmTargetClass = cdmTargetClass; |
85 | 85 |
} |
86 | 86 |
|
87 |
protected void doInvoke(GlobisImportState state){ |
|
87 |
@Override |
|
88 |
protected void doInvoke(GlobisImportState state){ |
|
88 | 89 |
logger.info("start make " + getPluralString() + " ..."); |
89 | 90 |
GlobisImportConfigurator config = state.getConfig(); |
90 | 91 |
Source source = config.getSource(); |
91 |
|
|
92 |
|
|
92 | 93 |
String strIdQuery = getIdQuery(); |
93 | 94 |
String strRecordQuery = getRecordQuery(config); |
94 | 95 |
|
... | ... | |
102 | 103 |
logger.error("SQLException:" + e); |
103 | 104 |
state.setUnsuccessfull(); |
104 | 105 |
} |
105 |
|
|
106 |
|
|
106 | 107 |
logger.info("end make " + getPluralString() + " ... " + getSuccessString(true)); |
107 | 108 |
return; |
108 | 109 |
} |
... | ... | |
124 | 125 |
doubtfulAuthorAndYear = authorAndYear; |
125 | 126 |
authorAndYear = authorAndYear.replace("[", "").replace("]", ""); |
126 | 127 |
} |
127 |
|
|
128 |
|
|
128 | 129 |
parser.parseAuthors(zooName, authorAndYear); |
129 | 130 |
deduplicateAuthors(zooName, state); |
130 |
|
|
131 |
|
|
131 | 132 |
if (doubtfulAuthorAndYear != null){ |
132 | 133 |
zooName.setAuthorshipCache(doubtfulAuthorAndYear, true); |
133 | 134 |
} |
134 |
|
|
135 |
|
|
135 | 136 |
} catch (StringNotParsableException e) { |
136 | 137 |
logger.warn("Author could not be parsed: " + authorAndYear + " for id " +id); |
137 | 138 |
zooName.setAuthorshipCache(authorAndYear, true); |
... | ... | |
152 | 153 |
state.putTeam(teamStr, team); |
153 | 154 |
getAgentService().save(team); |
154 | 155 |
} |
155 |
zooName.setCombinationAuthorTeam(team);
|
|
156 |
zooName.setCombinationAuthorship(team);
|
|
156 | 157 |
zooName.setPublicationYear(1775); |
157 | 158 |
zooName.setAuthorshipCache("[Denis & Schifferm\u00FCller], 1775", true); |
158 | 159 |
} |
159 |
|
|
160 |
|
|
160 | 161 |
|
161 | 162 |
private void deduplicateAuthors(ZoologicalName zooName, GlobisImportState state) { |
162 |
zooName.setCombinationAuthorTeam(getExistingAuthor(zooName.getCombinationAuthorTeam(), state));
|
|
163 |
zooName.setExCombinationAuthorTeam(getExistingAuthor(zooName.getExCombinationAuthorTeam(), state));
|
|
164 |
zooName.setBasionymAuthorTeam(getExistingAuthor(zooName.getBasionymAuthorTeam(), state));
|
|
165 |
zooName.setExBasionymAuthorTeam(getExistingAuthor(zooName.getExBasionymAuthorTeam(), state));
|
|
163 |
zooName.setCombinationAuthorship(getExistingAuthor(zooName.getCombinationAuthorship(), state));
|
|
164 |
zooName.setExCombinationAuthorship(getExistingAuthor(zooName.getExCombinationAuthorship(), state));
|
|
165 |
zooName.setBasionymAuthorship(getExistingAuthor(zooName.getBasionymAuthorship(), state));
|
|
166 |
zooName.setExBasionymAuthorship(getExistingAuthor(zooName.getExBasionymAuthorship(), state));
|
|
166 | 167 |
} |
167 | 168 |
|
168 | 169 |
private TeamOrPersonBase<?> getExistingAuthor(INomenclaturalAuthor nomAuthor, GlobisImportState state) { |
... | ... | |
184 | 185 |
try { |
185 | 186 |
getAgentService().update(existingPerson); |
186 | 187 |
} catch (NonUniqueObjectException nuoe){ |
187 |
// person already exists in
|
|
188 |
// person already exists in |
|
188 | 189 |
existingPerson = CdmBase.deproxy(getAgentService().find(existingPerson.getUuid()), Person.class); |
189 | 190 |
state.putPerson(existingPerson.getTitleCache(), existingPerson); |
190 | 191 |
} catch (Exception e) { |
191 | 192 |
throw new RuntimeException (e); |
192 | 193 |
} |
193 | 194 |
newTeam.addTeamMember(existingPerson); |
194 |
//
|
|
195 |
// |
|
195 | 196 |
// logger.warn("newTeam " + newTeam.getId()); |
196 | 197 |
}else{ |
197 | 198 |
newTeam.addTeamMember(member); |
198 |
}
|
|
199 |
} |
|
199 | 200 |
} |
200 | 201 |
author = newTeam; |
201 | 202 |
} |
202 |
|
|
203 |
|
|
203 | 204 |
return saveAndDecide(existingTeam, author, key, state); |
204 | 205 |
}else{ |
205 | 206 |
logger.warn("Author type not supported: " + author.getClass().getName()); |
... | ... | |
212 | 213 |
try { |
213 | 214 |
getAgentService().update(existing); |
214 | 215 |
} catch (NonUniqueObjectException nuoe){ |
215 |
// person already exists in
|
|
216 |
// person already exists in |
|
216 | 217 |
existing = CdmBase.deproxy(getAgentService().find(existing.getUuid()), TeamOrPersonBase.class); |
217 | 218 |
putAgent(existing, key, state); |
218 | 219 |
} catch (Exception e) { |
... | ... | |
248 | 249 |
NamedArea country = Country.getCountryByLabel(countryStr); |
249 | 250 |
if (country == null){ |
250 | 251 |
try { |
251 |
country = (NamedArea)state.getTransformer().getNamedAreaByKey(countryStr);
|
|
252 |
country = state.getTransformer().getNamedAreaByKey(countryStr); |
|
252 | 253 |
} catch (UndefinedTransformerMethodException e) { |
253 | 254 |
e.printStackTrace(); |
254 | 255 |
} |
... | ... | |
256 | 257 |
return country; |
257 | 258 |
} |
258 | 259 |
|
259 |
|
|
260 | 260 |
|
261 |
public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) { |
|
261 |
|
|
262 |
@Override |
|
263 |
public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) { |
|
262 | 264 |
boolean success = true ; |
263 | 265 |
Set objectsToSave = new HashSet(); |
264 |
|
|
266 |
|
|
265 | 267 |
DbImportMapping<?, ?> mapping = getMapping(); |
266 | 268 |
mapping.initialize(state, cdmTargetClass); |
267 |
|
|
269 |
|
|
268 | 270 |
ResultSet rs = partitioner.getResultSet(); |
269 | 271 |
try{ |
270 | 272 |
while (rs.next()){ |
... | ... | |
274 | 276 |
logger.error("SQLException:" + e); |
275 | 277 |
return false; |
276 | 278 |
} |
277 |
|
|
279 |
|
|
278 | 280 |
partitioner.startDoSave(); |
279 | 281 |
getCommonService().save(objectsToSave); |
280 | 282 |
return success; |
281 | 283 |
} |
282 | 284 |
|
283 | 285 |
|
284 |
|
|
286 |
|
|
285 | 287 |
/** |
286 | 288 |
* @return |
287 | 289 |
*/ |
288 | 290 |
protected /*abstract*/ DbImportMapping<?, ?> getMapping(){ |
289 | 291 |
return null; |
290 | 292 |
} |
291 |
|
|
293 |
|
|
292 | 294 |
/** |
293 | 295 |
* @return |
294 | 296 |
*/ |
... | ... | |
301 | 303 |
String result = " SELECT id FROM " + getTableName(); |
302 | 304 |
return result; |
303 | 305 |
} |
304 |
|
|
306 |
|
|
305 | 307 |
/* (non-Javadoc) |
306 | 308 |
* @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getPluralString() |
307 | 309 |
*/ |
308 |
public String getPluralString(){ |
|
310 |
@Override |
|
311 |
public String getPluralString(){ |
|
309 | 312 |
return pluralString; |
310 | 313 |
} |
311 | 314 |
|
... | ... | |
315 | 318 |
protected String getTableName(){ |
316 | 319 |
return this.dbTableName; |
317 | 320 |
} |
318 |
|
|
321 |
|
|
319 | 322 |
protected boolean doIdCreatedUpdatedNotes(GlobisImportState state, IdentifiableEntity identifiableEntity, ResultSet rs, long id, String namespace) |
320 | 323 |
throws SQLException{ |
321 | 324 |
boolean success = true; |
... | ... | |
325 | 328 |
success &= doCreatedUpdatedNotes(state, identifiableEntity, rs, namespace); |
326 | 329 |
return success; |
327 | 330 |
} |
328 |
|
|
329 |
|
|
331 |
|
|
332 |
|
|
330 | 333 |
protected boolean doCreatedUpdatedNotes(GlobisImportState state, AnnotatableEntity annotatableEntity, ResultSet rs, String namespace) |
331 | 334 |
throws SQLException{ |
332 | 335 |
|
... | ... | |
336 | 339 |
Object updatedWhen = rs.getObject("Updated_When"); |
337 | 340 |
String updatedWho = rs.getString("Updated_who"); |
338 | 341 |
String notes = rs.getString("notes"); |
339 |
|
|
342 |
|
|
340 | 343 |
boolean success = true; |
341 |
|
|
344 |
|
|
342 | 345 |
//Created When, Who, Updated When Who |
343 | 346 |
if (config.getEditor() == null || config.getEditor().equals(EDITOR.NO_EDITORS)){ |
344 | 347 |
//do nothing |
... | ... | |
363 | 366 |
}else { |
364 | 367 |
logger.warn("Editor type not yet implemented: " + config.getEditor()); |
365 | 368 |
} |
366 |
|
|
367 |
|
|
369 |
|
|
370 |
|
|
368 | 371 |
//notes |
369 | 372 |
if (StringUtils.isNotBlank(notes)){ |
370 | 373 |
String notesString = String.valueOf(notes); |
... | ... | |
379 | 382 |
} |
380 | 383 |
return success; |
381 | 384 |
} |
382 |
|
|
385 |
|
|
383 | 386 |
private User getUser(String userString, GlobisImportState state){ |
384 | 387 |
if (isBlank(userString)){ |
385 | 388 |
return null; |
386 | 389 |
} |
387 | 390 |
userString = userString.trim(); |
388 |
|
|
391 |
|
|
389 | 392 |
User user = state.getUser(userString); |
390 | 393 |
if (user == null){ |
391 | 394 |
user = getTransformedUser(userString,state); |
... | ... | |
398 | 401 |
} |
399 | 402 |
return user; |
400 | 403 |
} |
401 |
|
|
404 |
|
|
402 | 405 |
private User getTransformedUser(String userString, GlobisImportState state){ |
403 | 406 |
Method method = state.getConfig().getUserTransformationMethod(); |
404 | 407 |
if (method == null){ |
... | ... | |
414 | 417 |
} |
415 | 418 |
|
416 | 419 |
private User makeNewUser(String userString, GlobisImportState state){ |
417 |
String pwd = getPassword();
|
|
420 |
String pwd = getPassword(); |
|
418 | 421 |
User user = User.NewInstance(userString, pwd); |
419 | 422 |
state.putUser(userString, user); |
420 | 423 |
getUserService().save(user); |
421 | 424 |
logger.info("Added new user: " + userString); |
422 | 425 |
return user; |
423 | 426 |
} |
424 |
|
|
427 |
|
|
425 | 428 |
private String getPassword(){ |
426 | 429 |
String result = UUID.randomUUID().toString(); |
427 | 430 |
return result; |
428 | 431 |
} |
429 |
|
|
432 |
|
|
430 | 433 |
private DateTime getDateTime(Object timeString){ |
431 | 434 |
if (timeString == null){ |
432 | 435 |
return null; |
... | ... | |
441 | 444 |
} |
442 | 445 |
return dateTime; |
443 | 446 |
} |
444 |
|
|
445 |
|
|
446 |
|
|
447 |
|
|
448 |
|
|
449 |
|
|
447 | 450 |
/** |
448 | 451 |
* Reads a foreign key field from the result set and adds its value to the idSet. |
449 | 452 |
* @param rs |
... | ... | |
458 | 461 |
idSet.add(id); |
459 | 462 |
} |
460 | 463 |
} |
461 |
|
|
462 |
|
|
463 |
|
|
464 |
|
|
464 |
|
|
465 |
|
|
466 |
|
|
467 |
|
|
465 | 468 |
/** |
466 | 469 |
* Returns true if i is a multiple of recordsPerTransaction |
467 | 470 |
* @param i |
... | ... | |
472 | 475 |
startTransaction(); |
473 | 476 |
return (i % recordsPerLoop) == 0; |
474 | 477 |
} |
475 |
|
|
478 |
|
|
476 | 479 |
protected void doLogPerLoop(int count, int recordsPerLog, String pluralString){ |
477 | 480 |
if ((count % recordsPerLog ) == 0 && count!= 0 ){ logger.info(pluralString + " handled: " + (count));} |
478 | 481 |
} |
479 |
|
|
480 | 482 |
|
481 | 483 |
|
482 |
|
|
484 |
|
|
485 |
|
|
483 | 486 |
} |
Also available in: Unified diff
Rename NonViralName authorteams to authorship #4968