1
|
// $Id$
|
2
|
/**
|
3
|
* Copyright (C) 2017 EDIT
|
4
|
* European Distributed Institute of Taxonomy
|
5
|
* http://www.e-taxonomy.eu
|
6
|
*
|
7
|
* The contents of this file are subject to the Mozilla Public License Version 1.1
|
8
|
* See LICENSE.TXT at the top of this package for the full license terms.
|
9
|
*/
|
10
|
package eu.etaxonomy.cdm.io.bogota;
|
11
|
|
12
|
import java.util.Arrays;
|
13
|
import java.util.HashMap;
|
14
|
import java.util.List;
|
15
|
import java.util.Map;
|
16
|
import java.util.UUID;
|
17
|
|
18
|
import org.apache.log4j.Logger;
|
19
|
import org.joda.time.Partial;
|
20
|
import org.springframework.stereotype.Component;
|
21
|
|
22
|
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
|
23
|
import eu.etaxonomy.cdm.common.CdmUtils;
|
24
|
import eu.etaxonomy.cdm.io.common.utils.ImportDeduplicationHelper;
|
25
|
import eu.etaxonomy.cdm.model.agent.AgentBase;
|
26
|
import eu.etaxonomy.cdm.model.agent.Institution;
|
27
|
import eu.etaxonomy.cdm.model.agent.Person;
|
28
|
import eu.etaxonomy.cdm.model.agent.Team;
|
29
|
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
|
30
|
import eu.etaxonomy.cdm.model.common.CdmBase;
|
31
|
import eu.etaxonomy.cdm.model.common.ExtensionType;
|
32
|
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
|
33
|
import eu.etaxonomy.cdm.model.common.Language;
|
34
|
import eu.etaxonomy.cdm.model.common.TimePeriod;
|
35
|
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
|
36
|
import eu.etaxonomy.cdm.model.description.TaxonDescription;
|
37
|
import eu.etaxonomy.cdm.model.location.Country;
|
38
|
import eu.etaxonomy.cdm.model.location.NamedArea;
|
39
|
import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
|
40
|
import eu.etaxonomy.cdm.model.location.NamedAreaType;
|
41
|
import eu.etaxonomy.cdm.model.location.Point;
|
42
|
import eu.etaxonomy.cdm.model.location.ReferenceSystem;
|
43
|
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
|
44
|
import eu.etaxonomy.cdm.model.name.Rank;
|
45
|
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
|
46
|
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
|
47
|
import eu.etaxonomy.cdm.model.name.TaxonName;
|
48
|
import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
|
49
|
import eu.etaxonomy.cdm.model.occurrence.Collection;
|
50
|
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
|
51
|
import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
|
52
|
import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
|
53
|
import eu.etaxonomy.cdm.model.reference.Reference;
|
54
|
import eu.etaxonomy.cdm.model.taxon.Synonym;
|
55
|
import eu.etaxonomy.cdm.model.taxon.Taxon;
|
56
|
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
|
57
|
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
|
58
|
import eu.etaxonomy.cdm.model.term.DefinedTerm;
|
59
|
import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
|
60
|
import eu.etaxonomy.cdm.strategy.parser.DeterminationModifierParser;
|
61
|
import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
|
62
|
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
|
63
|
import eu.etaxonomy.cdm.strategy.parser.SpecimenTypeParser;
|
64
|
import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
|
65
|
|
66
|
/**
|
67
|
* @author a.mueller
|
68
|
* @since 21.04.2017
|
69
|
*
|
70
|
*/
|
71
|
@Component
|
72
|
public class BogotaSpecimenImport<CONFIG extends BogotaSpecimenImportConfigurator>
|
73
|
extends SimpleExcelSpecimenImport<CONFIG> {
|
74
|
|
75
|
private static final long serialVersionUID = -884838817884874228L;
|
76
|
@SuppressWarnings("unused")
|
77
|
private static final Logger logger = Logger.getLogger(BogotaSpecimenImport.class);
|
78
|
|
79
|
private static final String COL_TAXON_UUID = "Platform Name ID = cdmID";
|
80
|
private static final String COL_VOUCHER_ID = "Voucher ID";
|
81
|
|
82
|
private static final String COL_FAMILY = "Family";
|
83
|
private static final String COL_GENUS = "Genus";
|
84
|
private static final String COL_SPECIFIC_EPI = "Specific Epithet";
|
85
|
private static final String COL_BASIONYM_AUTHOR = "Author in parenthesis";
|
86
|
private static final String COL_AUTHOR = "Author";
|
87
|
private static final String COL_IDENTIFIER = "Identifier";
|
88
|
private static final String COL_IDENTIFICATION_DATE = "Identification date";
|
89
|
private static final String COL_IDENTIFICATION_QUALIFIER = "Qualifier";
|
90
|
private static final String COL_TYPUS = "Type";
|
91
|
private static final String COL_IDENTIFICATION_HISTORY = "Identification history";
|
92
|
private static final String COL_COLLECTOR_VERBATIM = "Verbatim Collectors (Originalfeld JBB)";
|
93
|
private static final String COL_COLLECTOR_LASTNAME = "Primary Collector Last Name (Originalfeld JBB)";
|
94
|
private static final String COL_COLLECTOR_FIRSTNAME = "Primary Collector First Name Initial (Originalfeld JBB)";
|
95
|
private static final String COL_COLLECTOR_MIDDLENAME = "Primary Collector Middle Name Initial (Originalfeld JBB)";
|
96
|
private static final String COL_COLLECTOR_TYPE = "Primary collector type (Originalfeld JBB)";
|
97
|
private static final String COL_COLLECTOR_NUMBER = "Collector's No";
|
98
|
private static final String COL_COLLECTORS = "Collectors";
|
99
|
private static final String COL_COLLECTION_DATE_FROM = "Collection Date from";
|
100
|
private static final String COL_COLLECTION_DATE_TO = "Collection Date to";
|
101
|
private static final String COL_ALTITUDE_FROM = "Altitude Value from";
|
102
|
private static final String COL_ALTITUDE_TO = "Altitude Value to";
|
103
|
private static final String COL_ALTITUDE_UNIT = "Altitude Unit";
|
104
|
private static final String COL_LOCALITY = "Locality";
|
105
|
private static final String COL_LOCALITY_ID = "LocalityID";
|
106
|
private static final String COL_LATITUDE = "Latitude";
|
107
|
private static final String COL_LONGITUDE = "Longitude";
|
108
|
private static final String COL_ERROR_DISTANCE = "Error distance in m";
|
109
|
private static final String COL_COUNTRY = "Country";
|
110
|
private static final String COL_STATE_AREA = "State/Province/Greater Area";
|
111
|
private static final String COL_GEO_METHOD = "Geocode Method";
|
112
|
private static final String COL_HABITUS = "Habitus";
|
113
|
private static final String COL_COLLECTION = "[Series] Voucher location";
|
114
|
|
115
|
private static final UUID uuidAnonymous = UUID.fromString("2303f043-6e92-4afa-9082-7719e78a1c8a");
|
116
|
private static final UUID uuidBogota = UUID.fromString("95b6cb03-8452-4439-98bd-8c1aa3c1da4e");
|
117
|
private static final UUID uuidDefaultGeocodMethod = UUID.fromString("0983e680-b0ca-4e46-8df7-0f1d757a2e01");
|
118
|
private static final UUID uuidExtTypeIdentificationHistory = UUID.fromString("7cee5c29-e16b-4e6f-ad57-bf7044259375");
|
119
|
private static final UUID uuidDetQualVelAff = UUID.fromString("511a0c23-2646-4035-b570-36bdc2eb5557");
|
120
|
|
121
|
// @SuppressWarnings("unchecked")
|
122
|
private ImportDeduplicationHelper<SimpleExcelSpecimenImportState<?>> deduplicationHelper;
|
123
|
|
124
|
private final Map<String, TaxonNode> taxonNodeMap = new HashMap<>();
|
125
|
private Reference secRef;
|
126
|
|
127
|
@Override
|
128
|
protected String getWorksheetName(CONFIG config) {
|
129
|
return "To be imported";
|
130
|
}
|
131
|
|
132
|
/**
|
133
|
* {@inheritDoc}
|
134
|
*/
|
135
|
@Override
|
136
|
protected void firstPass(SimpleExcelSpecimenImportState<CONFIG> state) {
|
137
|
|
138
|
Map<String, String> record = state.getOriginalRecord();
|
139
|
|
140
|
String voucherId = getValue(record, COL_VOUCHER_ID);
|
141
|
if (!isInInterval(state)){
|
142
|
return;
|
143
|
}
|
144
|
String line = state.getCurrentLine() + " (id:"+ voucherId+"): ";
|
145
|
if (state.getCurrentLine() % 100 == 0){System.out.println(line);}
|
146
|
try {
|
147
|
|
148
|
//species
|
149
|
TaxonBase<?> taxonBase = getTaxonByCdmId(state, line, record, voucherId);
|
150
|
if (taxonBase != null){
|
151
|
handleRecordForTaxon(state, voucherId, line, taxonBase);
|
152
|
}else if (record.get(COL_TAXON_UUID)!= null){
|
153
|
// do nothing
|
154
|
}else{
|
155
|
taxonBase = getOrCreateNewTaxon(state, record, line);
|
156
|
handleRecordForTaxon(state, voucherId, line, taxonBase);
|
157
|
}
|
158
|
|
159
|
} catch (Exception e) {
|
160
|
state.getResult().addError("An unexpected exception appeared in record", e, null, line);
|
161
|
e.printStackTrace();
|
162
|
}
|
163
|
|
164
|
}
|
165
|
|
166
|
/**
|
167
|
* @param state
|
168
|
* @param record
|
169
|
* @param line
|
170
|
* @return
|
171
|
*/
|
172
|
private Taxon getOrCreateNewTaxon(SimpleExcelSpecimenImportState<CONFIG> state,
|
173
|
Map<String, String> record, String line) {
|
174
|
String familyStr = record.get(COL_FAMILY);
|
175
|
String genusStr = record.get(COL_GENUS);
|
176
|
initTaxonMap(state);
|
177
|
TaxonName speciesName = makeSpeciesName(state, line);
|
178
|
String titleCache = speciesName.getTitleCache();
|
179
|
TaxonNode existingSpeciesNode = taxonNodeMap.get(titleCache);
|
180
|
if (existingSpeciesNode != null){
|
181
|
return existingSpeciesNode.getTaxon();
|
182
|
}else{
|
183
|
Reference sec = getSecReference(state);
|
184
|
Taxon newTaxon = Taxon.NewInstance(speciesName, sec);
|
185
|
newTaxon.addSource(makeOriginalSource(state));
|
186
|
TaxonNode existingGenusNode = taxonNodeMap.get(genusStr);
|
187
|
if (existingGenusNode == null){
|
188
|
TaxonName genusName = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
|
189
|
genusName.setGenusOrUninomial(genusStr);
|
190
|
Taxon newGenus = Taxon.NewInstance(genusName, sec);
|
191
|
newGenus.addSource(makeOriginalSource(state));
|
192
|
TaxonNode existingFamilyNode = taxonNodeMap.get(familyStr);
|
193
|
if (existingFamilyNode == null){
|
194
|
TaxonName familyName = TaxonNameFactory.NewBotanicalInstance(Rank.FAMILY());
|
195
|
familyName.setGenusOrUninomial(familyStr);
|
196
|
Taxon newFamily = Taxon.NewInstance(familyName, sec);
|
197
|
newFamily.addSource(makeOriginalSource(state));
|
198
|
TaxonNode plantaeNode = taxonNodeMap.get("Plantae");
|
199
|
existingFamilyNode = plantaeNode.addChildTaxon(newFamily, null, null);
|
200
|
save(existingFamilyNode);
|
201
|
}
|
202
|
existingGenusNode = existingFamilyNode.addChildTaxon(newGenus, null, null);
|
203
|
save(existingGenusNode);
|
204
|
}
|
205
|
existingSpeciesNode = existingGenusNode.addChildTaxon(newTaxon, null, null);
|
206
|
save(existingSpeciesNode);
|
207
|
return newTaxon;
|
208
|
}
|
209
|
|
210
|
}
|
211
|
|
212
|
/**
|
213
|
* @param existingFamilyNode
|
214
|
*/
|
215
|
private void save(TaxonNode node) {
|
216
|
getTaxonNodeService().saveOrUpdate(node);
|
217
|
taxonNodeMap.put(node.getTaxon().getName().getTitleCache(), node);
|
218
|
|
219
|
}
|
220
|
|
221
|
/**
|
222
|
* @param state
|
223
|
* @return
|
224
|
*/
|
225
|
private Reference getSecReference(SimpleExcelSpecimenImportState<CONFIG> state) {
|
226
|
if (this.secRef == null){
|
227
|
Reference sec = state.getConfig().getSecReference();
|
228
|
this.secRef = getReferenceService().find(sec.getUuid());
|
229
|
if (this.secRef == null){
|
230
|
this.secRef = sec;
|
231
|
getReferenceService().save(sec);
|
232
|
}
|
233
|
}
|
234
|
|
235
|
|
236
|
return this.secRef;
|
237
|
}
|
238
|
|
239
|
/**
|
240
|
* @param state
|
241
|
* @param record
|
242
|
* @param line
|
243
|
* @return
|
244
|
*/
|
245
|
private TaxonName makeSpeciesName(SimpleExcelSpecimenImportState<CONFIG> state, String line) {
|
246
|
Map<String, String> record = state.getOriginalRecord();
|
247
|
String genus = record.get(COL_GENUS);
|
248
|
String species = record.get(COL_SPECIFIC_EPI);
|
249
|
String basionymAuthorStr = record.get(COL_BASIONYM_AUTHOR);
|
250
|
String authorStr = record.get(COL_AUTHOR);
|
251
|
INonViralNameParser<?> parser = NonViralNameParserImpl.NewInstance();
|
252
|
String fullName = genus + " " + species +
|
253
|
(basionymAuthorStr == null ? "" : " ("+basionymAuthorStr+")")
|
254
|
+ " " + authorStr;
|
255
|
TaxonName newName = (TaxonName)parser.parseFullName(fullName , NomenclaturalCode.ICNAFP, Rank.SPECIES());
|
256
|
String titleCache = newName.getTitleCache();
|
257
|
if (newName.isProtectedTitleCache()){
|
258
|
state.getResult().addWarning("Name not parsable: " + fullName);
|
259
|
}
|
260
|
if (taxonNodeMap.get(titleCache)== null){
|
261
|
getDeduplicationHelper(state).replaceAuthorNamesAndNomRef(state, newName);
|
262
|
newName.addSource(makeOriginalSource(state));
|
263
|
}
|
264
|
|
265
|
return newName;
|
266
|
}
|
267
|
|
268
|
/**
|
269
|
* @param state
|
270
|
*
|
271
|
*/
|
272
|
private void initTaxonMap(SimpleExcelSpecimenImportState<CONFIG> state) {
|
273
|
if (taxonNodeMap.isEmpty()){
|
274
|
List<String> propertyPaths = Arrays.asList(new String[]{"taxon.name"});
|
275
|
List<TaxonNode> list = getTaxonNodeService().list(null, null, null, null, propertyPaths);
|
276
|
for (TaxonNode node : list){
|
277
|
if (node.getTaxon()!= null){
|
278
|
String strName = node.getTaxon().getName().getTitleCache();
|
279
|
TaxonNode existingNode = taxonNodeMap.get(strName);
|
280
|
if (existingNode != null){
|
281
|
state.getResult().addWarning("Taxon name exists more than once while initializing taxon map: " + strName, "initTaxonMap");
|
282
|
}else{
|
283
|
taxonNodeMap.put(strName, node);
|
284
|
}
|
285
|
}
|
286
|
}
|
287
|
}
|
288
|
}
|
289
|
|
290
|
/**
|
291
|
* @param state
|
292
|
* @param record
|
293
|
* @param voucherId
|
294
|
* @param line
|
295
|
* @param taxonBase
|
296
|
* @param taxon
|
297
|
*/
|
298
|
protected void handleRecordForTaxon(SimpleExcelSpecimenImportState<CONFIG> state,
|
299
|
String voucherId, String line, TaxonBase<?> taxonBase) {
|
300
|
|
301
|
Map<String, String> record = state.getOriginalRecord();
|
302
|
Taxon taxon = getTaxon(taxonBase);
|
303
|
|
304
|
TaxonDescription taxonDescription = getTaxonDescription(state, line, taxon);
|
305
|
|
306
|
DerivedUnit specimen = makeSpecimen(state, line, record, voucherId, taxonBase);
|
307
|
|
308
|
IndividualsAssociation indAssoc = IndividualsAssociation.NewInstance(specimen);
|
309
|
indAssoc.addImportSource(voucherId, COL_VOUCHER_ID, getSourceCitation(state), null);
|
310
|
taxonDescription.addElement(indAssoc);
|
311
|
}
|
312
|
|
313
|
|
314
|
/**
|
315
|
* @param state
|
316
|
* @return
|
317
|
*/
|
318
|
private boolean isInInterval(SimpleExcelSpecimenImportState<CONFIG> state) {
|
319
|
Integer min = state.getConfig().getMinLineNumber();
|
320
|
Integer max = state.getConfig().getMaxLineNumber();
|
321
|
Integer current = state.getCurrentLine();
|
322
|
if (current < min || current > max){
|
323
|
return false;
|
324
|
}else{
|
325
|
return true;
|
326
|
}
|
327
|
}
|
328
|
|
329
|
|
330
|
/**
|
331
|
* @param state
|
332
|
* @param line
|
333
|
* @param taxon
|
334
|
* @return
|
335
|
*/
|
336
|
private TaxonDescription getTaxonDescription(SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
337
|
Taxon taxon) {
|
338
|
Reference ref = getSourceCitation(state);
|
339
|
TaxonDescription desc = this.getTaxonDescription(taxon, ref, ! IMAGE_GALLERY, ! CREATE);
|
340
|
if (desc == null){
|
341
|
//TODO move title creation into base method
|
342
|
desc = this.getTaxonDescription(taxon, ref, ! IMAGE_GALLERY, CREATE);
|
343
|
desc.setTitleCache("Specimen Excel import for " + taxon.getName().getTitleCache(), true);
|
344
|
}
|
345
|
return desc;
|
346
|
}
|
347
|
|
348
|
|
349
|
/**
|
350
|
* @param state
|
351
|
* @param line
|
352
|
* @param record
|
353
|
* @param voucherId
|
354
|
* @return
|
355
|
*/
|
356
|
private DerivedUnit makeSpecimen(SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
357
|
Map<String, String> record, String voucherId, TaxonBase<?> taxonBase) {
|
358
|
|
359
|
DerivedUnitFacade facade = DerivedUnitFacade.NewPreservedSpecimenInstance();
|
360
|
facade.setAccessionNumber(voucherId);
|
361
|
makeDetermination(facade.innerDerivedUnit(), state, line, record, taxonBase.getName());
|
362
|
makeTypus(facade.innerDerivedUnit(), state, line, record, taxonBase.getName());
|
363
|
makeCollectorFields(facade, state, line, record);
|
364
|
makeLocationFields(facade, state, line, record);
|
365
|
makeHabitus(facade, state, line, record);
|
366
|
makeCollection(facade, state, line, record);
|
367
|
DerivedUnit specimen = facade.innerDerivedUnit();
|
368
|
specimen.addSource(makeOriginalSource(state));
|
369
|
FieldUnit fieldUnit = facade.innerFieldUnit();
|
370
|
fieldUnit.addSource(makeOriginalSource(state));
|
371
|
return specimen;
|
372
|
}
|
373
|
|
374
|
|
375
|
/**
|
376
|
* @param innerDerivedUnit
|
377
|
* @param state
|
378
|
* @param line
|
379
|
* @param record
|
380
|
* @param name
|
381
|
*/
|
382
|
private void makeTypus(DerivedUnit specimen, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
383
|
Map<String, String> record, TaxonName name) {
|
384
|
String typus = record.get(COL_TYPUS);
|
385
|
if (typus != null){
|
386
|
SpecimenTypeDesignationStatus status;
|
387
|
try {
|
388
|
status = SpecimenTypeParser.parseSpecimenTypeStatus(typus);
|
389
|
SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
|
390
|
designation.setTypeStatus(status);
|
391
|
name.addSpecimenTypeDesignation(specimen, status, null, null, null, false, false);
|
392
|
} catch (UnknownCdmTypeException e) {
|
393
|
state.getResult().addWarning("Type designation could not be parsed", null, line);
|
394
|
}
|
395
|
}
|
396
|
}
|
397
|
|
398
|
|
399
|
/**
|
400
|
* @param facade
|
401
|
* @param state
|
402
|
* @param line
|
403
|
* @param record
|
404
|
*/
|
405
|
private void makeCollection(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state,
|
406
|
String line, Map<String, String> record) {
|
407
|
String strCollection = record.get(COL_COLLECTION);
|
408
|
String collectionFormat = ".*\\([A-Z]{2,4}\\)";
|
409
|
if (!strCollection.matches(collectionFormat)){
|
410
|
String message = "Voucher location format does not match the expected format. Voucher '(" + strCollection + ")' location not added.";
|
411
|
state.getResult().addError(message, null, line);
|
412
|
return;
|
413
|
}
|
414
|
String[] splits = strCollection.split("\\(");
|
415
|
String collectionName = splits[0];
|
416
|
String collectionCode = splits[1].replace(")", "");
|
417
|
Collection collection = Collection.NewInstance();
|
418
|
collection.setName(collectionName);
|
419
|
collection.setCode(collectionCode);
|
420
|
collection = getDeduplicationHelper(state).getExistingCollection(state, collection);
|
421
|
facade.setCollection(collection);
|
422
|
}
|
423
|
|
424
|
|
425
|
/**
|
426
|
* @param state
|
427
|
* @return
|
428
|
*/
|
429
|
private ImportDeduplicationHelper<SimpleExcelSpecimenImportState<?>> getDeduplicationHelper(SimpleExcelSpecimenImportState<CONFIG> state) {
|
430
|
if (deduplicationHelper == null){
|
431
|
deduplicationHelper = ImportDeduplicationHelper.NewInstance(this, state);
|
432
|
}
|
433
|
return deduplicationHelper;
|
434
|
}
|
435
|
|
436
|
|
437
|
/**
|
438
|
* @param facade
|
439
|
* @param state
|
440
|
* @param line
|
441
|
* @param record
|
442
|
*/
|
443
|
private void makeHabitus(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
444
|
Map<String, String> record) {
|
445
|
String habitus = record.get(COL_HABITUS);
|
446
|
if (habitus != null){
|
447
|
facade.setPlantDescription(habitus);
|
448
|
}
|
449
|
}
|
450
|
|
451
|
|
452
|
/**
|
453
|
* @param facade
|
454
|
* @param state
|
455
|
* @param line
|
456
|
* @param record
|
457
|
* @param voucherId
|
458
|
*/
|
459
|
private void makeLocationFields(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
460
|
Map<String, String> record) {
|
461
|
//Altitude
|
462
|
String strAltitudeFrom = record.get(COL_ALTITUDE_FROM);
|
463
|
String strAltitudeTo = record.get(COL_ALTITUDE_TO);
|
464
|
Integer intAltitudeFrom = intFromString(state, strAltitudeFrom, line, COL_ALTITUDE_FROM);
|
465
|
Integer intAltitudeTo = intFromString(state, strAltitudeTo, line, COL_ALTITUDE_TO);
|
466
|
if (intAltitudeFrom != null){
|
467
|
facade.setAbsoluteElevation(intAltitudeFrom);
|
468
|
if (!intAltitudeFrom.equals(intAltitudeTo)){
|
469
|
facade.setAbsoluteElevationMax(intAltitudeTo);
|
470
|
}
|
471
|
if (!record.get(COL_ALTITUDE_UNIT).equals("m")){
|
472
|
state.getResult().addWarning("Altitude unit is not m but " + record.get(COL_ALTITUDE_UNIT), "makeLocationFields", line);
|
473
|
}
|
474
|
}
|
475
|
checkNoToIfNoFrom(strAltitudeFrom, strAltitudeTo, state, line, COL_ALTITUDE_TO);
|
476
|
|
477
|
//locality
|
478
|
String locality = record.get(COL_LOCALITY);
|
479
|
if (locality != null){ //should always exist
|
480
|
facade.setLocality(locality, Language.SPANISH_CASTILIAN());
|
481
|
}
|
482
|
|
483
|
//Lat + Long
|
484
|
String strLatitude = record.get(COL_LATITUDE);
|
485
|
String strLongitude = record.get(COL_LONGITUDE);
|
486
|
String strError = record.get(COL_ERROR_DISTANCE);
|
487
|
Double dblLatitude = doubleFromString(state, strLatitude, line, COL_LATITUDE);
|
488
|
Double dblLongitude = doubleFromString(state, strLongitude, line, COL_LONGITUDE);
|
489
|
Integer intError = intFromString(state, strError, line, COL_ERROR_DISTANCE);
|
490
|
ReferenceSystem referenceSystem = makeReferenceSystem(state, record, line);
|
491
|
|
492
|
if (dblLatitude != null || dblLongitude != null || intError != null){ //should always exist
|
493
|
Point exactLocation = Point.NewInstance(dblLongitude, dblLatitude, referenceSystem, intError);
|
494
|
facade.setExactLocation(exactLocation);
|
495
|
}
|
496
|
|
497
|
//Country
|
498
|
String strCountry = record.get(COL_COUNTRY);
|
499
|
if (strCountry != null){
|
500
|
if (strCountry.equals("Colombia")){
|
501
|
Country colombia = Country.COLOMBIAREPUBLICOF();
|
502
|
colombia.setLabel("Colombia");
|
503
|
getTermService().saveOrUpdate(colombia);
|
504
|
facade.setCountry(colombia);
|
505
|
}else{
|
506
|
state.getResult().addWarning("Country was not Colombia as expected but " + strCountry,
|
507
|
"makeLocationFields", line);
|
508
|
}
|
509
|
}
|
510
|
|
511
|
//State
|
512
|
String strStateArea = record.get(COL_STATE_AREA);
|
513
|
if (strStateArea != null){
|
514
|
if (strStateArea.replaceAll("\\s", "").equalsIgnoreCase("Bogotá,D.C.")){
|
515
|
NamedArea bogota = makeBogota(state, line);
|
516
|
facade.addCollectingArea(bogota);
|
517
|
}else{
|
518
|
state.getResult().addWarning(COL_STATE_AREA + " was not 'Bogotá, D.C.' as expected but " + strCountry,
|
519
|
"makeLocationFields", line);
|
520
|
}
|
521
|
}
|
522
|
}
|
523
|
|
524
|
|
525
|
/**
|
526
|
* @param strAltitudeFrom
|
527
|
* @param strAltitudeTo
|
528
|
* @param state
|
529
|
* @param line
|
530
|
* @param colAltitudeTo
|
531
|
*/
|
532
|
private void checkNoToIfNoFrom(String strFrom, String strTo,
|
533
|
SimpleExcelSpecimenImportState<CONFIG> state,
|
534
|
String line, String toAttributeName) {
|
535
|
if (isNotBlank(strTo) && isBlank(strFrom)){
|
536
|
String message = "A min-max attribute has a max value (%s) but no min value. This is invalid."
|
537
|
+ " The max value attribute name is %s.";
|
538
|
message = String.format(message, strTo, toAttributeName);
|
539
|
state.getResult().addWarning(message, null, line);
|
540
|
}
|
541
|
}
|
542
|
|
543
|
private ReferenceSystem defaultGeocodeMethod;
|
544
|
|
545
|
/**
|
546
|
* @param state
|
547
|
* @param record
|
548
|
* @param line
|
549
|
* @return
|
550
|
*/
|
551
|
private ReferenceSystem makeReferenceSystem(SimpleExcelSpecimenImportState<CONFIG> state,
|
552
|
Map<String, String> record, String line) {
|
553
|
String defaultStrRefSys = "Wieczorek, J., Guo, Q., & Hijmans, R. (2004). The point-radius method for georeferencing locality descriptions and calculating associated uncertainty. International journal of geographical information science, 18(8), 745-767.; Escobar D, Díaz SR, Jojoa LM, Rudas E, Albarracín RD, Ramírez C, Gómez JY, López CR, Saavedra J (2015). Georreferenciación de localidades: Una guía de referencia para colecciones biológicas. Instituto de Investigación de Recursos Biológicos Alexander von Humboldt – Instituto de Ciencias Naturales, Universidad Nacional de Colombia. Bogotá D.C., Colombia. 95 p.";
|
554
|
String strRefSys = record.get(COL_GEO_METHOD);
|
555
|
if (strRefSys == null){
|
556
|
return null;
|
557
|
}else if (!strRefSys.equals(defaultStrRefSys)){
|
558
|
state.getResult().addError("The expected Geocode Method is not the expected default method. Geocode Method was not added.", null, line);
|
559
|
return null;
|
560
|
}else if (defaultGeocodeMethod != null){
|
561
|
return defaultGeocodeMethod;
|
562
|
}else{
|
563
|
String label = "Point radius method";
|
564
|
String description = defaultStrRefSys;
|
565
|
String labelAbbrev = "PRM";
|
566
|
defaultGeocodeMethod = getReferenceSystem(state, uuidDefaultGeocodMethod,
|
567
|
label, description, labelAbbrev, null);
|
568
|
return defaultGeocodeMethod;
|
569
|
}
|
570
|
}
|
571
|
|
572
|
private NamedArea bogota;
|
573
|
/**
|
574
|
* @param state
|
575
|
* @param line
|
576
|
* @return
|
577
|
*/
|
578
|
private NamedArea makeBogota(SimpleExcelSpecimenImportState<CONFIG> state, String line) {
|
579
|
if (bogota != null){
|
580
|
return bogota;
|
581
|
}else{
|
582
|
String label = "Bogotá, D.C.";
|
583
|
NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
|
584
|
NamedAreaLevel level = NamedAreaLevel.STATE();
|
585
|
bogota = getNamedArea(state, uuidBogota, label, label, null, areaType,
|
586
|
level, null, null, null);
|
587
|
return bogota;
|
588
|
}
|
589
|
}
|
590
|
|
591
|
|
592
|
/**
|
593
|
* @param facade
|
594
|
* @param state
|
595
|
* @param line
|
596
|
* @param record
|
597
|
*/
|
598
|
private void makeCollectorFields(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
599
|
Map<String, String> record) {
|
600
|
|
601
|
//collector number
|
602
|
facade.setFieldNumber(record.get(COL_COLLECTOR_NUMBER));
|
603
|
|
604
|
//gathering date
|
605
|
String dateFrom = unknownToNull((record.get(COL_COLLECTION_DATE_FROM)));
|
606
|
String dateTo = unknownToNull(record.get(COL_COLLECTION_DATE_TO));
|
607
|
checkNoToIfNoFrom(dateFrom, dateTo, state, line, COL_COLLECTION_DATE_TO);
|
608
|
try {
|
609
|
if (dateFrom != null && dateFrom.equals(dateTo)){
|
610
|
dateTo = null;
|
611
|
}
|
612
|
TimePeriod gatheringPeriod = TimePeriodParser.parseEnglishDate(dateFrom, dateTo);
|
613
|
facade.setGatheringPeriod(gatheringPeriod);
|
614
|
} catch (Exception e) {
|
615
|
state.getResult().addError("Error creating gathering date", e, null, line);
|
616
|
}
|
617
|
|
618
|
//collector
|
619
|
String collectorType = record.get(COL_COLLECTOR_TYPE);
|
620
|
String collectors = record.get(COL_COLLECTORS);
|
621
|
AgentBase<?> collector;
|
622
|
if (collectorType.startsWith("Anonymous")){
|
623
|
collector = getAnonymous();
|
624
|
}else if (collectorType.equals("Brother") || collectorType.equals("Person")){
|
625
|
Person person = Person.NewInstance();
|
626
|
if (collectorType.equals("Person")){
|
627
|
person.setFamilyName(record.get(COL_COLLECTOR_LASTNAME));
|
628
|
String initials = CdmUtils.concat("", record.get(COL_COLLECTOR_FIRSTNAME), record.get(COL_COLLECTOR_MIDDLENAME));
|
629
|
initials = (initials == null)? null : initials.replaceAll("\\s", "");
|
630
|
person.setInitials(initials);
|
631
|
String full = person.getTitleCache();
|
632
|
if (!full.equals(collectors)){
|
633
|
person.setTitleCache(collectors, true);
|
634
|
//TODO use setCollectorTitle in future
|
635
|
}
|
636
|
}else{
|
637
|
person.setTitleCache(collectors, true);
|
638
|
person.setPrefix("Hno.");
|
639
|
person.setGivenName(collectors.replace("Hno.", "").trim());
|
640
|
}
|
641
|
collector = person;
|
642
|
}else if (collectorType.equals("Group")){
|
643
|
collector = Team.NewTitledInstance(collectors, collectors);
|
644
|
}else if (collectorType.equals("Institution")){
|
645
|
collector = Institution.NewNamedInstance(collectors);
|
646
|
}else{
|
647
|
String message = "Collector type " + collectorType + " not yet supported by import. Collector not added.";
|
648
|
state.getResult().addError(message, null, line);
|
649
|
collector = null;
|
650
|
}
|
651
|
collector = getDeduplicationHelper(state).getExistingAgent(state, collector);
|
652
|
facade.setCollector(collector);
|
653
|
}
|
654
|
|
655
|
|
656
|
/**
|
657
|
* @param string
|
658
|
* @return
|
659
|
*/
|
660
|
private String unknownToNull(String string) {
|
661
|
if (string == null || string.equalsIgnoreCase("unknown")){
|
662
|
return null;
|
663
|
}else{
|
664
|
return string;
|
665
|
}
|
666
|
}
|
667
|
|
668
|
private Person anonymous;
|
669
|
private Person getAnonymous() {
|
670
|
if (anonymous != null){
|
671
|
return anonymous;
|
672
|
}
|
673
|
anonymous = CdmBase.deproxy(getAgentService().find(uuidAnonymous), Person.class);
|
674
|
if (anonymous == null){
|
675
|
anonymous = Person.NewTitledInstance("Anon.");
|
676
|
anonymous.setUuid(uuidAnonymous);
|
677
|
getAgentService().save(anonymous);
|
678
|
}
|
679
|
return anonymous;
|
680
|
}
|
681
|
|
682
|
|
683
|
/**
|
684
|
* @param facade
|
685
|
* @param state
|
686
|
* @param line
|
687
|
* @param record
|
688
|
* @param taxonBase
|
689
|
*/
|
690
|
private void makeDetermination(DerivedUnit specimen, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
691
|
Map<String, String> record, TaxonName taxonName) {
|
692
|
|
693
|
DeterminationEvent determination;
|
694
|
determination = DeterminationEvent.NewInstance(taxonName, specimen);
|
695
|
determination.setPreferredFlag(true);
|
696
|
|
697
|
//determiner/identifier
|
698
|
TeamOrPersonBase<?> determiner = makeDeterminer(state, record, line);
|
699
|
determination.setDeterminer(determiner);
|
700
|
|
701
|
//date
|
702
|
TimePeriod date = makeIdentificationDate(state, record, line);
|
703
|
determination.setTimeperiod(date);
|
704
|
|
705
|
//qualifier
|
706
|
DefinedTerm qualifier = makeDeterminationQualifier(state, record, line);
|
707
|
determination.setModifier(qualifier);
|
708
|
|
709
|
//history
|
710
|
String history = record.get(COL_IDENTIFICATION_HISTORY);
|
711
|
if (history != null){
|
712
|
String label = "Identification History";
|
713
|
String text = label;
|
714
|
ExtensionType detHistoryType = getExtensionType(state, uuidExtTypeIdentificationHistory, label, text, null);
|
715
|
specimen.addExtension(history, detHistoryType);
|
716
|
}
|
717
|
}
|
718
|
|
719
|
|
720
|
/**
|
721
|
* @param state
|
722
|
* @param record
|
723
|
* @param line
|
724
|
* @return
|
725
|
*/
|
726
|
private TeamOrPersonBase<?> makeDeterminer(SimpleExcelSpecimenImportState<CONFIG> state,
|
727
|
Map<String, String> record, String line) {
|
728
|
String identifier = record.get(COL_IDENTIFIER);
|
729
|
if (identifier == null){
|
730
|
return null;
|
731
|
}else if (identifier.equals("Anon.")){
|
732
|
return getAnonymous();
|
733
|
}else{
|
734
|
Person person = Person.NewInstance();
|
735
|
person.setTitleCache(identifier, true);
|
736
|
|
737
|
String[] splits = identifier.split("\\.");
|
738
|
int length = splits.length;
|
739
|
if (splits[length - 1].equals("")){
|
740
|
splits[length - 2]= splits[length - 2]+".";
|
741
|
length--;
|
742
|
}
|
743
|
if (splits[length - 1].startsWith("-")){
|
744
|
splits[length - 2]= splits[length - 2]+"." + splits[length - 1];
|
745
|
length--;
|
746
|
}
|
747
|
String familyName = splits[length - 1];
|
748
|
String initials = null;
|
749
|
for (int i= 0; i < length-1;i++){
|
750
|
initials = CdmUtils.concat("", initials, splits[i]+".");
|
751
|
}
|
752
|
person.setFamilyName(familyName);
|
753
|
person.setInitials(initials);
|
754
|
TeamOrPersonBase<?> result = getDeduplicationHelper(state).getExistingAuthor(state, person);
|
755
|
return result;
|
756
|
}
|
757
|
}
|
758
|
|
759
|
|
760
|
/**
|
761
|
* @param state
|
762
|
* @param record
|
763
|
* @param line
|
764
|
* @return
|
765
|
*/
|
766
|
private TimePeriod makeIdentificationDate(SimpleExcelSpecimenImportState<CONFIG> state,
|
767
|
Map<String, String> record, String line) {
|
768
|
String strDate = record.get(COL_IDENTIFICATION_DATE);
|
769
|
if (strDate == null || strDate.equals("s.n.")){
|
770
|
return null;
|
771
|
}
|
772
|
String[] splits = strDate.split("/");
|
773
|
String strYear = splits[splits.length-1];
|
774
|
String strMonth = splits.length < 2? null:splits[splits.length-2];
|
775
|
String strDay = splits.length < 3? null:splits[splits.length-3];
|
776
|
|
777
|
Integer year = intFromString(state, strYear, line, COL_IDENTIFICATION_DATE);
|
778
|
Integer month = intFromString(state, strMonth, line, COL_IDENTIFICATION_DATE);
|
779
|
Integer day = intFromString(state, strDay, line, COL_IDENTIFICATION_DATE);
|
780
|
Partial start = TimePeriodParser.makePartialFromDateParts(year, month, day);
|
781
|
return TimePeriod.NewInstance(start);
|
782
|
}
|
783
|
|
784
|
|
785
|
/**
|
786
|
* @param state
|
787
|
* @param record
|
788
|
* @param line
|
789
|
* @return
|
790
|
*/
|
791
|
private DefinedTerm makeDeterminationQualifier(SimpleExcelSpecimenImportState<CONFIG> state,
|
792
|
Map<String, String> record, String line) {
|
793
|
String qualifier = record.get(COL_IDENTIFICATION_QUALIFIER);
|
794
|
if (qualifier != null){
|
795
|
try {
|
796
|
return DeterminationModifierParser.parseDeterminationQualifier(qualifier);
|
797
|
} catch (UnknownCdmTypeException e) {
|
798
|
//TODO add to terms
|
799
|
if (qualifier.equals("vel. aff.")){
|
800
|
|
801
|
DefinedTerm velAff = (DefinedTerm)getTermService().find(uuidDetQualVelAff);
|
802
|
if (velAff == null){
|
803
|
velAff = DefinedTerm.NewModifierInstance(qualifier, qualifier, qualifier);
|
804
|
velAff.setUuid(uuidDetQualVelAff);
|
805
|
getTermService().save(velAff);
|
806
|
}
|
807
|
return velAff;
|
808
|
}
|
809
|
state.getResult().addError("Determination qualifier could not be recognized: " + qualifier, null, line);
|
810
|
return null;
|
811
|
}
|
812
|
}else{
|
813
|
return null;
|
814
|
}
|
815
|
}
|
816
|
|
817
|
|
818
|
/**
|
819
|
* @param taxonBase
|
820
|
* @return
|
821
|
*/
|
822
|
private Taxon getTaxon(TaxonBase<?> taxonBase) {
|
823
|
if (taxonBase.isInstanceOf(Synonym.class)){
|
824
|
return CdmBase.deproxy(taxonBase, Synonym.class).getAcceptedTaxon();
|
825
|
}else{
|
826
|
return CdmBase.deproxy(taxonBase, Taxon.class);
|
827
|
}
|
828
|
}
|
829
|
|
830
|
|
831
|
/**
|
832
|
* @param state
|
833
|
* @param line
|
834
|
* @param record
|
835
|
* @param noStr
|
836
|
* @return
|
837
|
*/
|
838
|
private TaxonBase<?> getTaxonByCdmId(SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
839
|
Map<String, String> record, String noStr) {
|
840
|
|
841
|
String strUuidTaxon = record.get(COL_TAXON_UUID);
|
842
|
if (strUuidTaxon != null && ! state.getConfig().isOnlyNonCdmTaxa()){
|
843
|
UUID uuidTaxon;
|
844
|
try {
|
845
|
uuidTaxon = UUID.fromString(strUuidTaxon);
|
846
|
} catch (Exception e) {
|
847
|
state.getResult().addError("Taxon uuid has incorrect format. Taxon could not be loaded. Data not imported.", null, line);
|
848
|
return null;
|
849
|
}
|
850
|
TaxonBase<?> result = getTaxonService().find(uuidTaxon);
|
851
|
if (result == null){
|
852
|
state.getResult().addError("Taxon for uuid "+strUuidTaxon+" could not be found in database. "
|
853
|
+ "Taxon could not be loaded. Data not imported.", null, line);
|
854
|
|
855
|
}
|
856
|
return result;
|
857
|
}else{
|
858
|
return null;
|
859
|
}
|
860
|
}
|
861
|
|
862
|
@Override
|
863
|
protected IdentifiableSource makeOriginalSource(SimpleExcelSpecimenImportState<CONFIG> state) {
|
864
|
return IdentifiableSource.NewDataImportInstance(getValue(state.getOriginalRecord(), COL_VOUCHER_ID), COL_VOUCHER_ID, getSourceCitation(state));
|
865
|
}
|
866
|
|
867
|
/**
|
868
|
* @param state
|
869
|
* @return
|
870
|
*/
|
871
|
protected Reference getSourceCitation(SimpleExcelSpecimenImportState<CONFIG> state) {
|
872
|
Reference source = state.getConfig().getSourceReference();
|
873
|
if (source.getId() == 0){
|
874
|
Reference persisted = getReferenceService().find(source.getUuid());
|
875
|
if (persisted == null){
|
876
|
getReferenceService().saveOrUpdate(source);
|
877
|
}else{
|
878
|
state.getConfig().setSourceReference(persisted);
|
879
|
source = persisted;
|
880
|
}
|
881
|
}
|
882
|
return source;
|
883
|
}
|
884
|
}
|