1
|
/**
|
2
|
* Copyright (C) 2017 EDIT
|
3
|
* European Distributed Institute of Taxonomy
|
4
|
* http://www.e-taxonomy.eu
|
5
|
*
|
6
|
* The contents of this file are subject to the Mozilla Public License Version 1.1
|
7
|
* See LICENSE.TXT at the top of this package for the full license terms.
|
8
|
*/
|
9
|
package eu.etaxonomy.cdm.io.bogota;
|
10
|
|
11
|
import java.util.Arrays;
|
12
|
import java.util.HashMap;
|
13
|
import java.util.List;
|
14
|
import java.util.Map;
|
15
|
import java.util.UUID;
|
16
|
|
17
|
import org.apache.logging.log4j.LogManager;
|
18
|
import org.apache.logging.log4j.Logger;
|
19
|
import org.joda.time.Partial;
|
20
|
import org.springframework.stereotype.Component;
|
21
|
|
22
|
import eu.etaxonomy.cdm.common.CdmUtils;
|
23
|
import eu.etaxonomy.cdm.facade.DerivedUnitFacade;
|
24
|
import eu.etaxonomy.cdm.model.agent.AgentBase;
|
25
|
import eu.etaxonomy.cdm.model.agent.Institution;
|
26
|
import eu.etaxonomy.cdm.model.agent.Person;
|
27
|
import eu.etaxonomy.cdm.model.agent.Team;
|
28
|
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
|
29
|
import eu.etaxonomy.cdm.model.common.CdmBase;
|
30
|
import eu.etaxonomy.cdm.model.common.ExtensionType;
|
31
|
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
|
32
|
import eu.etaxonomy.cdm.model.common.Language;
|
33
|
import eu.etaxonomy.cdm.model.common.TimePeriod;
|
34
|
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
|
35
|
import eu.etaxonomy.cdm.model.description.TaxonDescription;
|
36
|
import eu.etaxonomy.cdm.model.location.Country;
|
37
|
import eu.etaxonomy.cdm.model.location.NamedArea;
|
38
|
import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
|
39
|
import eu.etaxonomy.cdm.model.location.NamedAreaType;
|
40
|
import eu.etaxonomy.cdm.model.location.Point;
|
41
|
import eu.etaxonomy.cdm.model.location.ReferenceSystem;
|
42
|
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
|
43
|
import eu.etaxonomy.cdm.model.name.Rank;
|
44
|
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
|
45
|
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
|
46
|
import eu.etaxonomy.cdm.model.name.TaxonName;
|
47
|
import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
|
48
|
import eu.etaxonomy.cdm.model.occurrence.Collection;
|
49
|
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
|
50
|
import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
|
51
|
import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
|
52
|
import eu.etaxonomy.cdm.model.reference.Reference;
|
53
|
import eu.etaxonomy.cdm.model.taxon.Synonym;
|
54
|
import eu.etaxonomy.cdm.model.taxon.Taxon;
|
55
|
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
|
56
|
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
|
57
|
import eu.etaxonomy.cdm.model.term.DefinedTerm;
|
58
|
import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
|
59
|
import eu.etaxonomy.cdm.strategy.parser.DeterminationModifierParser;
|
60
|
import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
|
61
|
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
|
62
|
import eu.etaxonomy.cdm.strategy.parser.SpecimenTypeParser;
|
63
|
import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
|
64
|
|
65
|
/**
|
66
|
* @author a.mueller
|
67
|
* @since 21.04.2017
|
68
|
*/
|
69
|
@Component
|
70
|
public class BogotaSpecimenImport<CONFIG extends BogotaSpecimenImportConfigurator>
|
71
|
extends SimpleExcelSpecimenImport<CONFIG> {
|
72
|
|
73
|
private static final long serialVersionUID = -884838817884874228L;
|
74
|
@SuppressWarnings("unused")
|
75
|
private static final Logger logger = LogManager.getLogger();
|
76
|
|
77
|
private static final String COL_TAXON_UUID = "Platform Name ID = cdmID";
|
78
|
private static final String COL_VOUCHER_ID = "Voucher ID";
|
79
|
|
80
|
private static final String COL_FAMILY = "Family";
|
81
|
private static final String COL_GENUS = "Genus";
|
82
|
private static final String COL_SPECIFIC_EPI = "Specific Epithet";
|
83
|
private static final String COL_BASIONYM_AUTHOR = "Author in parenthesis";
|
84
|
private static final String COL_AUTHOR = "Author";
|
85
|
private static final String COL_IDENTIFIER = "Identifier";
|
86
|
private static final String COL_IDENTIFICATION_DATE = "Identification date";
|
87
|
private static final String COL_IDENTIFICATION_QUALIFIER = "Qualifier";
|
88
|
private static final String COL_TYPUS = "Type";
|
89
|
private static final String COL_IDENTIFICATION_HISTORY = "Identification history";
|
90
|
private static final String COL_COLLECTOR_VERBATIM = "Verbatim Collectors (Originalfeld JBB)";
|
91
|
private static final String COL_COLLECTOR_LASTNAME = "Primary Collector Last Name (Originalfeld JBB)";
|
92
|
private static final String COL_COLLECTOR_FIRSTNAME = "Primary Collector First Name Initial (Originalfeld JBB)";
|
93
|
private static final String COL_COLLECTOR_MIDDLENAME = "Primary Collector Middle Name Initial (Originalfeld JBB)";
|
94
|
private static final String COL_COLLECTOR_TYPE = "Primary collector type (Originalfeld JBB)";
|
95
|
private static final String COL_COLLECTOR_NUMBER = "Collector's No";
|
96
|
private static final String COL_COLLECTORS = "Collectors";
|
97
|
private static final String COL_COLLECTION_DATE_FROM = "Collection Date from";
|
98
|
private static final String COL_COLLECTION_DATE_TO = "Collection Date to";
|
99
|
private static final String COL_ALTITUDE_FROM = "Altitude Value from";
|
100
|
private static final String COL_ALTITUDE_TO = "Altitude Value to";
|
101
|
private static final String COL_ALTITUDE_UNIT = "Altitude Unit";
|
102
|
private static final String COL_LOCALITY = "Locality";
|
103
|
private static final String COL_LOCALITY_ID = "LocalityID";
|
104
|
private static final String COL_LATITUDE = "Latitude";
|
105
|
private static final String COL_LONGITUDE = "Longitude";
|
106
|
private static final String COL_ERROR_DISTANCE = "Error distance in m";
|
107
|
private static final String COL_COUNTRY = "Country";
|
108
|
private static final String COL_STATE_AREA = "State/Province/Greater Area";
|
109
|
private static final String COL_GEO_METHOD = "Geocode Method";
|
110
|
private static final String COL_HABITUS = "Habitus";
|
111
|
private static final String COL_COLLECTION = "[Series] Voucher location";
|
112
|
|
113
|
private static final UUID uuidAnonymous = UUID.fromString("2303f043-6e92-4afa-9082-7719e78a1c8a");
|
114
|
private static final UUID uuidBogota = UUID.fromString("95b6cb03-8452-4439-98bd-8c1aa3c1da4e");
|
115
|
private static final UUID uuidDefaultGeocodMethod = UUID.fromString("0983e680-b0ca-4e46-8df7-0f1d757a2e01");
|
116
|
private static final UUID uuidExtTypeIdentificationHistory = UUID.fromString("7cee5c29-e16b-4e6f-ad57-bf7044259375");
|
117
|
private static final UUID uuidDetQualVelAff = UUID.fromString("511a0c23-2646-4035-b570-36bdc2eb5557");
|
118
|
|
119
|
private final Map<String, TaxonNode> taxonNodeMap = new HashMap<>();
|
120
|
private Reference secRef;
|
121
|
|
122
|
@Override
|
123
|
protected String getWorksheetName(CONFIG config) {
|
124
|
return "To be imported";
|
125
|
}
|
126
|
|
127
|
/**
|
128
|
* {@inheritDoc}
|
129
|
*/
|
130
|
@Override
|
131
|
protected void firstPass(SimpleExcelSpecimenImportState<CONFIG> state) {
|
132
|
|
133
|
Map<String, String> record = state.getOriginalRecord();
|
134
|
|
135
|
String voucherId = getValue(record, COL_VOUCHER_ID);
|
136
|
if (!isInInterval(state)){
|
137
|
return;
|
138
|
}
|
139
|
String line = state.getCurrentLine() + " (id:"+ voucherId+"): ";
|
140
|
if (state.getCurrentLine() % 100 == 0){System.out.println(line);}
|
141
|
try {
|
142
|
|
143
|
//species
|
144
|
TaxonBase<?> taxonBase = getTaxonByCdmId(state, line, record, voucherId);
|
145
|
if (taxonBase != null){
|
146
|
handleRecordForTaxon(state, voucherId, line, taxonBase);
|
147
|
}else if (record.get(COL_TAXON_UUID)!= null){
|
148
|
// do nothing
|
149
|
}else{
|
150
|
taxonBase = getOrCreateNewTaxon(state, record, line);
|
151
|
handleRecordForTaxon(state, voucherId, line, taxonBase);
|
152
|
}
|
153
|
|
154
|
} catch (Exception e) {
|
155
|
state.getResult().addError("An unexpected exception appeared in record", e, null, line);
|
156
|
e.printStackTrace();
|
157
|
}
|
158
|
|
159
|
}
|
160
|
|
161
|
/**
|
162
|
* @param state
|
163
|
* @param record
|
164
|
* @param line
|
165
|
* @return
|
166
|
*/
|
167
|
private Taxon getOrCreateNewTaxon(SimpleExcelSpecimenImportState<CONFIG> state,
|
168
|
Map<String, String> record, String line) {
|
169
|
String familyStr = record.get(COL_FAMILY);
|
170
|
String genusStr = record.get(COL_GENUS);
|
171
|
initTaxonMap(state);
|
172
|
TaxonName speciesName = makeSpeciesName(state, line);
|
173
|
String titleCache = speciesName.getTitleCache();
|
174
|
TaxonNode existingSpeciesNode = taxonNodeMap.get(titleCache);
|
175
|
if (existingSpeciesNode != null){
|
176
|
return existingSpeciesNode.getTaxon();
|
177
|
}else{
|
178
|
Reference sec = getSecReference(state);
|
179
|
Taxon newTaxon = Taxon.NewInstance(speciesName, sec);
|
180
|
newTaxon.addSource(makeOriginalSource(state));
|
181
|
TaxonNode existingGenusNode = taxonNodeMap.get(genusStr);
|
182
|
if (existingGenusNode == null){
|
183
|
TaxonName genusName = TaxonNameFactory.NewBotanicalInstance(Rank.GENUS());
|
184
|
genusName.setGenusOrUninomial(genusStr);
|
185
|
Taxon newGenus = Taxon.NewInstance(genusName, sec);
|
186
|
newGenus.addSource(makeOriginalSource(state));
|
187
|
TaxonNode existingFamilyNode = taxonNodeMap.get(familyStr);
|
188
|
if (existingFamilyNode == null){
|
189
|
TaxonName familyName = TaxonNameFactory.NewBotanicalInstance(Rank.FAMILY());
|
190
|
familyName.setGenusOrUninomial(familyStr);
|
191
|
Taxon newFamily = Taxon.NewInstance(familyName, sec);
|
192
|
newFamily.addSource(makeOriginalSource(state));
|
193
|
TaxonNode plantaeNode = taxonNodeMap.get("Plantae");
|
194
|
existingFamilyNode = plantaeNode.addChildTaxon(newFamily, null, null);
|
195
|
save(existingFamilyNode);
|
196
|
}
|
197
|
existingGenusNode = existingFamilyNode.addChildTaxon(newGenus, null, null);
|
198
|
save(existingGenusNode);
|
199
|
}
|
200
|
existingSpeciesNode = existingGenusNode.addChildTaxon(newTaxon, null, null);
|
201
|
save(existingSpeciesNode);
|
202
|
return newTaxon;
|
203
|
}
|
204
|
|
205
|
}
|
206
|
|
207
|
/**
|
208
|
* @param existingFamilyNode
|
209
|
*/
|
210
|
private void save(TaxonNode node) {
|
211
|
getTaxonNodeService().saveOrUpdate(node);
|
212
|
taxonNodeMap.put(node.getTaxon().getName().getTitleCache(), node);
|
213
|
|
214
|
}
|
215
|
|
216
|
/**
|
217
|
* @param state
|
218
|
* @return
|
219
|
*/
|
220
|
private Reference getSecReference(SimpleExcelSpecimenImportState<CONFIG> state) {
|
221
|
if (this.secRef == null){
|
222
|
Reference sec = state.getConfig().getSecReference();
|
223
|
this.secRef = getReferenceService().find(sec.getUuid());
|
224
|
if (this.secRef == null){
|
225
|
this.secRef = sec;
|
226
|
getReferenceService().save(sec);
|
227
|
}
|
228
|
}
|
229
|
|
230
|
|
231
|
return this.secRef;
|
232
|
}
|
233
|
|
234
|
/**
|
235
|
* @param state
|
236
|
* @param record
|
237
|
* @param line
|
238
|
* @return
|
239
|
*/
|
240
|
private TaxonName makeSpeciesName(SimpleExcelSpecimenImportState<CONFIG> state, String line) {
|
241
|
Map<String, String> record = state.getOriginalRecord();
|
242
|
String genus = record.get(COL_GENUS);
|
243
|
String species = record.get(COL_SPECIFIC_EPI);
|
244
|
String basionymAuthorStr = record.get(COL_BASIONYM_AUTHOR);
|
245
|
String authorStr = record.get(COL_AUTHOR);
|
246
|
INonViralNameParser<?> parser = NonViralNameParserImpl.NewInstance();
|
247
|
String fullName = genus + " " + species +
|
248
|
(basionymAuthorStr == null ? "" : " ("+basionymAuthorStr+")")
|
249
|
+ " " + authorStr;
|
250
|
TaxonName newName = (TaxonName)parser.parseFullName(fullName , NomenclaturalCode.ICNAFP, Rank.SPECIES());
|
251
|
String titleCache = newName.getTitleCache();
|
252
|
if (newName.isProtectedTitleCache()){
|
253
|
state.getResult().addWarning("Name not parsable: " + fullName);
|
254
|
}
|
255
|
if (taxonNodeMap.get(titleCache)== null){
|
256
|
state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(newName);
|
257
|
newName.addSource(makeOriginalSource(state));
|
258
|
}
|
259
|
|
260
|
return newName;
|
261
|
}
|
262
|
|
263
|
/**
|
264
|
* @param state
|
265
|
*
|
266
|
*/
|
267
|
private void initTaxonMap(SimpleExcelSpecimenImportState<CONFIG> state) {
|
268
|
if (taxonNodeMap.isEmpty()){
|
269
|
List<String> propertyPaths = Arrays.asList(new String[]{"taxon.name"});
|
270
|
List<TaxonNode> list = getTaxonNodeService().list(null, null, null, null, propertyPaths);
|
271
|
for (TaxonNode node : list){
|
272
|
if (node.getTaxon()!= null){
|
273
|
String strName = node.getTaxon().getName().getTitleCache();
|
274
|
TaxonNode existingNode = taxonNodeMap.get(strName);
|
275
|
if (existingNode != null){
|
276
|
state.getResult().addWarning("Taxon name exists more than once while initializing taxon map: " + strName, "initTaxonMap");
|
277
|
}else{
|
278
|
taxonNodeMap.put(strName, node);
|
279
|
}
|
280
|
}
|
281
|
}
|
282
|
}
|
283
|
}
|
284
|
|
285
|
/**
|
286
|
* @param state
|
287
|
* @param record
|
288
|
* @param voucherId
|
289
|
* @param line
|
290
|
* @param taxonBase
|
291
|
* @param taxon
|
292
|
*/
|
293
|
protected void handleRecordForTaxon(SimpleExcelSpecimenImportState<CONFIG> state,
|
294
|
String voucherId, String line, TaxonBase<?> taxonBase) {
|
295
|
|
296
|
Map<String, String> record = state.getOriginalRecord();
|
297
|
Taxon taxon = getTaxon(taxonBase);
|
298
|
|
299
|
TaxonDescription taxonDescription = getTaxonDescription(state, line, taxon);
|
300
|
|
301
|
DerivedUnit specimen = makeSpecimen(state, line, record, voucherId, taxonBase);
|
302
|
|
303
|
IndividualsAssociation indAssoc = IndividualsAssociation.NewInstance(specimen);
|
304
|
indAssoc.addImportSource(voucherId, COL_VOUCHER_ID, getSourceCitation(state), null);
|
305
|
taxonDescription.addElement(indAssoc);
|
306
|
}
|
307
|
|
308
|
|
309
|
/**
|
310
|
* @param state
|
311
|
* @return
|
312
|
*/
|
313
|
private boolean isInInterval(SimpleExcelSpecimenImportState<CONFIG> state) {
|
314
|
Integer min = state.getConfig().getMinLineNumber();
|
315
|
Integer max = state.getConfig().getMaxLineNumber();
|
316
|
Integer current = state.getCurrentLine();
|
317
|
if (current < min || current > max){
|
318
|
return false;
|
319
|
}else{
|
320
|
return true;
|
321
|
}
|
322
|
}
|
323
|
|
324
|
|
325
|
/**
|
326
|
* @param state
|
327
|
* @param line
|
328
|
* @param taxon
|
329
|
* @return
|
330
|
*/
|
331
|
private TaxonDescription getTaxonDescription(SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
332
|
Taxon taxon) {
|
333
|
Reference ref = getSourceCitation(state);
|
334
|
TaxonDescription desc = this.getTaxonDescription(taxon, ref, ! IMAGE_GALLERY, ! CREATE);
|
335
|
if (desc == null){
|
336
|
//TODO move title creation into base method
|
337
|
desc = this.getTaxonDescription(taxon, ref, ! IMAGE_GALLERY, CREATE);
|
338
|
desc.setTitleCache("Specimen Excel import for " + taxon.getName().getTitleCache(), true);
|
339
|
}
|
340
|
return desc;
|
341
|
}
|
342
|
|
343
|
|
344
|
/**
|
345
|
* @param state
|
346
|
* @param line
|
347
|
* @param record
|
348
|
* @param voucherId
|
349
|
* @return
|
350
|
*/
|
351
|
private DerivedUnit makeSpecimen(SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
352
|
Map<String, String> record, String voucherId, TaxonBase<?> taxonBase) {
|
353
|
|
354
|
DerivedUnitFacade facade = DerivedUnitFacade.NewPreservedSpecimenInstance();
|
355
|
facade.setAccessionNumber(voucherId);
|
356
|
makeDetermination(facade.innerDerivedUnit(), state, line, record, taxonBase.getName());
|
357
|
makeTypus(facade.innerDerivedUnit(), state, line, record, taxonBase.getName());
|
358
|
makeCollectorFields(facade, state, line, record);
|
359
|
makeLocationFields(facade, state, line, record);
|
360
|
makeHabitus(facade, state, line, record);
|
361
|
makeCollection(facade, state, line, record);
|
362
|
DerivedUnit specimen = facade.innerDerivedUnit();
|
363
|
specimen.addSource(makeOriginalSource(state));
|
364
|
FieldUnit fieldUnit = facade.innerFieldUnit();
|
365
|
fieldUnit.addSource(makeOriginalSource(state));
|
366
|
return specimen;
|
367
|
}
|
368
|
|
369
|
|
370
|
/**
|
371
|
* @param innerDerivedUnit
|
372
|
* @param state
|
373
|
* @param line
|
374
|
* @param record
|
375
|
* @param name
|
376
|
*/
|
377
|
private void makeTypus(DerivedUnit specimen, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
378
|
Map<String, String> record, TaxonName name) {
|
379
|
String typus = record.get(COL_TYPUS);
|
380
|
if (typus != null){
|
381
|
SpecimenTypeDesignationStatus status;
|
382
|
try {
|
383
|
status = SpecimenTypeParser.parseSpecimenTypeStatus(typus);
|
384
|
SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
|
385
|
designation.setTypeStatus(status);
|
386
|
name.addSpecimenTypeDesignation(specimen, status, null, null, null, false, false);
|
387
|
} catch (UnknownCdmTypeException e) {
|
388
|
state.getResult().addWarning("Type designation could not be parsed", null, line);
|
389
|
}
|
390
|
}
|
391
|
}
|
392
|
|
393
|
|
394
|
/**
|
395
|
* @param facade
|
396
|
* @param state
|
397
|
* @param line
|
398
|
* @param record
|
399
|
*/
|
400
|
private void makeCollection(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state,
|
401
|
String line, Map<String, String> record) {
|
402
|
String strCollection = record.get(COL_COLLECTION);
|
403
|
String collectionFormat = ".*\\([A-Z]{2,4}\\)";
|
404
|
if (!strCollection.matches(collectionFormat)){
|
405
|
String message = "Voucher location format does not match the expected format. Voucher '(" + strCollection + ")' location not added.";
|
406
|
state.getResult().addError(message, null, line);
|
407
|
return;
|
408
|
}
|
409
|
String[] splits = strCollection.split("\\(");
|
410
|
String collectionName = splits[0];
|
411
|
String collectionCode = splits[1].replace(")", "");
|
412
|
Collection collection = Collection.NewInstance();
|
413
|
collection.setName(collectionName);
|
414
|
collection.setCode(collectionCode);
|
415
|
collection = state.getDeduplicationHelper().getExistingCollection(collection);
|
416
|
facade.setCollection(collection);
|
417
|
}
|
418
|
|
419
|
private void makeHabitus(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
420
|
Map<String, String> record) {
|
421
|
String habitus = record.get(COL_HABITUS);
|
422
|
if (habitus != null){
|
423
|
facade.setPlantDescription(habitus);
|
424
|
}
|
425
|
}
|
426
|
|
427
|
private void makeLocationFields(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
428
|
Map<String, String> record) {
|
429
|
//Altitude
|
430
|
String strAltitudeFrom = record.get(COL_ALTITUDE_FROM);
|
431
|
String strAltitudeTo = record.get(COL_ALTITUDE_TO);
|
432
|
Integer intAltitudeFrom = intFromString(state, strAltitudeFrom, line, COL_ALTITUDE_FROM);
|
433
|
Integer intAltitudeTo = intFromString(state, strAltitudeTo, line, COL_ALTITUDE_TO);
|
434
|
if (intAltitudeFrom != null){
|
435
|
facade.setAbsoluteElevation(intAltitudeFrom);
|
436
|
if (!intAltitudeFrom.equals(intAltitudeTo)){
|
437
|
facade.setAbsoluteElevationMax(intAltitudeTo);
|
438
|
}
|
439
|
if (!record.get(COL_ALTITUDE_UNIT).equals("m")){
|
440
|
state.getResult().addWarning("Altitude unit is not m but " + record.get(COL_ALTITUDE_UNIT), "makeLocationFields", line);
|
441
|
}
|
442
|
}
|
443
|
checkNoToIfNoFrom(strAltitudeFrom, strAltitudeTo, state, line, COL_ALTITUDE_TO);
|
444
|
|
445
|
//locality
|
446
|
String locality = record.get(COL_LOCALITY);
|
447
|
if (locality != null){ //should always exist
|
448
|
facade.setLocality(locality, Language.SPANISH_CASTILIAN());
|
449
|
}
|
450
|
|
451
|
//Lat + Long
|
452
|
String strLatitude = record.get(COL_LATITUDE);
|
453
|
String strLongitude = record.get(COL_LONGITUDE);
|
454
|
String strError = record.get(COL_ERROR_DISTANCE);
|
455
|
Double dblLatitude = doubleFromString(state, strLatitude, line, COL_LATITUDE);
|
456
|
Double dblLongitude = doubleFromString(state, strLongitude, line, COL_LONGITUDE);
|
457
|
Integer intError = intFromString(state, strError, line, COL_ERROR_DISTANCE);
|
458
|
ReferenceSystem referenceSystem = makeReferenceSystem(state, record, line);
|
459
|
|
460
|
if (dblLatitude != null || dblLongitude != null || intError != null){ //should always exist
|
461
|
Point exactLocation = Point.NewInstance(dblLongitude, dblLatitude, referenceSystem, intError);
|
462
|
facade.setExactLocation(exactLocation);
|
463
|
}
|
464
|
|
465
|
//Country
|
466
|
String strCountry = record.get(COL_COUNTRY);
|
467
|
if (strCountry != null){
|
468
|
if (strCountry.equals("Colombia")){
|
469
|
Country colombia = Country.COLOMBIAREPUBLICOF();
|
470
|
colombia.setLabel("Colombia");
|
471
|
getTermService().saveOrUpdate(colombia);
|
472
|
facade.setCountry(colombia);
|
473
|
}else{
|
474
|
state.getResult().addWarning("Country was not Colombia as expected but " + strCountry,
|
475
|
"makeLocationFields", line);
|
476
|
}
|
477
|
}
|
478
|
|
479
|
//State
|
480
|
String strStateArea = record.get(COL_STATE_AREA);
|
481
|
if (strStateArea != null){
|
482
|
if (strStateArea.replaceAll("\\s", "").equalsIgnoreCase("Bogotá,D.C.")){
|
483
|
NamedArea bogota = makeBogota(state, line);
|
484
|
facade.addCollectingArea(bogota);
|
485
|
}else{
|
486
|
state.getResult().addWarning(COL_STATE_AREA + " was not 'Bogotá, D.C.' as expected but " + strCountry,
|
487
|
"makeLocationFields", line);
|
488
|
}
|
489
|
}
|
490
|
}
|
491
|
|
492
|
private void checkNoToIfNoFrom(String strFrom, String strTo,
|
493
|
SimpleExcelSpecimenImportState<CONFIG> state,
|
494
|
String line, String toAttributeName) {
|
495
|
if (isNotBlank(strTo) && isBlank(strFrom)){
|
496
|
String message = "A min-max attribute has a max value (%s) but no min value. This is invalid."
|
497
|
+ " The max value attribute name is %s.";
|
498
|
message = String.format(message, strTo, toAttributeName);
|
499
|
state.getResult().addWarning(message, null, line);
|
500
|
}
|
501
|
}
|
502
|
|
503
|
private ReferenceSystem defaultGeocodeMethod;
|
504
|
|
505
|
private ReferenceSystem makeReferenceSystem(SimpleExcelSpecimenImportState<CONFIG> state,
|
506
|
Map<String, String> record, String line) {
|
507
|
String defaultStrRefSys = "Wieczorek, J., Guo, Q., & Hijmans, R. (2004). The point-radius method for georeferencing locality descriptions and calculating associated uncertainty. International journal of geographical information science, 18(8), 745-767.; Escobar D, Díaz SR, Jojoa LM, Rudas E, Albarracín RD, Ramírez C, Gómez JY, López CR, Saavedra J (2015). Georreferenciación de localidades: Una guía de referencia para colecciones biológicas. Instituto de Investigación de Recursos Biológicos Alexander von Humboldt – Instituto de Ciencias Naturales, Universidad Nacional de Colombia. Bogotá D.C., Colombia. 95 p.";
|
508
|
String strRefSys = record.get(COL_GEO_METHOD);
|
509
|
if (strRefSys == null){
|
510
|
return null;
|
511
|
}else if (!strRefSys.equals(defaultStrRefSys)){
|
512
|
state.getResult().addError("The expected Geocode Method is not the expected default method. Geocode Method was not added.", null, line);
|
513
|
return null;
|
514
|
}else if (defaultGeocodeMethod != null){
|
515
|
return defaultGeocodeMethod;
|
516
|
}else{
|
517
|
String label = "Point radius method";
|
518
|
String description = defaultStrRefSys;
|
519
|
String labelAbbrev = "PRM";
|
520
|
defaultGeocodeMethod = getReferenceSystem(state, uuidDefaultGeocodMethod,
|
521
|
label, description, labelAbbrev, null);
|
522
|
return defaultGeocodeMethod;
|
523
|
}
|
524
|
}
|
525
|
|
526
|
private NamedArea bogota;
|
527
|
private NamedArea makeBogota(SimpleExcelSpecimenImportState<CONFIG> state, String line) {
|
528
|
if (bogota != null){
|
529
|
return bogota;
|
530
|
}else{
|
531
|
String label = "Bogotá, D.C.";
|
532
|
NamedAreaType areaType = NamedAreaType.ADMINISTRATION_AREA();
|
533
|
NamedAreaLevel level = NamedAreaLevel.STATE();
|
534
|
bogota = getNamedArea(state, uuidBogota, label, label, null, areaType,
|
535
|
level, null, null, null);
|
536
|
return bogota;
|
537
|
}
|
538
|
}
|
539
|
|
540
|
private void makeCollectorFields(DerivedUnitFacade facade, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
541
|
Map<String, String> record) {
|
542
|
|
543
|
//collector number
|
544
|
facade.setFieldNumber(record.get(COL_COLLECTOR_NUMBER));
|
545
|
|
546
|
//gathering date
|
547
|
String dateFrom = unknownToNull((record.get(COL_COLLECTION_DATE_FROM)));
|
548
|
String dateTo = unknownToNull(record.get(COL_COLLECTION_DATE_TO));
|
549
|
checkNoToIfNoFrom(dateFrom, dateTo, state, line, COL_COLLECTION_DATE_TO);
|
550
|
try {
|
551
|
if (dateFrom != null && dateFrom.equals(dateTo)){
|
552
|
dateTo = null;
|
553
|
}
|
554
|
TimePeriod gatheringPeriod = TimePeriodParser.parseEnglishDate(dateFrom, dateTo);
|
555
|
facade.setGatheringPeriod(gatheringPeriod);
|
556
|
} catch (Exception e) {
|
557
|
state.getResult().addError("Error creating gathering date", e, null, line);
|
558
|
}
|
559
|
|
560
|
//collector
|
561
|
String collectorType = record.get(COL_COLLECTOR_TYPE);
|
562
|
String collectors = record.get(COL_COLLECTORS);
|
563
|
AgentBase<?> collector;
|
564
|
if (collectorType.startsWith("Anonymous")){
|
565
|
collector = getAnonymous();
|
566
|
}else if (collectorType.equals("Brother") || collectorType.equals("Person")){
|
567
|
Person person = Person.NewInstance();
|
568
|
if (collectorType.equals("Person")){
|
569
|
person.setFamilyName(record.get(COL_COLLECTOR_LASTNAME));
|
570
|
String initials = CdmUtils.concat("", record.get(COL_COLLECTOR_FIRSTNAME), record.get(COL_COLLECTOR_MIDDLENAME));
|
571
|
initials = (initials == null)? null : initials.replaceAll("\\s", "");
|
572
|
person.setInitials(initials);
|
573
|
String full = person.getTitleCache();
|
574
|
if (!full.equals(collectors)){
|
575
|
person.setTitleCache(collectors, true);
|
576
|
//TODO use setCollectorTitle in future
|
577
|
}
|
578
|
}else{
|
579
|
person.setTitleCache(collectors, true);
|
580
|
person.setPrefix("Hno.");
|
581
|
person.setGivenName(collectors.replace("Hno.", "").trim());
|
582
|
}
|
583
|
collector = person;
|
584
|
}else if (collectorType.equals("Group")){
|
585
|
collector = Team.NewTitledInstance(collectors, collectors);
|
586
|
}else if (collectorType.equals("Institution")){
|
587
|
collector = Institution.NewNamedInstance(collectors);
|
588
|
}else{
|
589
|
String message = "Collector type " + collectorType + " not yet supported by import. Collector not added.";
|
590
|
state.getResult().addError(message, null, line);
|
591
|
collector = null;
|
592
|
}
|
593
|
collector = state.getDeduplicationHelper().getExistingAgent(collector, false);
|
594
|
facade.setCollector(collector);
|
595
|
}
|
596
|
|
597
|
private String unknownToNull(String string) {
|
598
|
if (string == null || string.equalsIgnoreCase("unknown")){
|
599
|
return null;
|
600
|
}else{
|
601
|
return string;
|
602
|
}
|
603
|
}
|
604
|
|
605
|
private Person anonymous;
|
606
|
private Person getAnonymous() {
|
607
|
if (anonymous != null){
|
608
|
return anonymous;
|
609
|
}
|
610
|
anonymous = CdmBase.deproxy(getAgentService().find(uuidAnonymous), Person.class);
|
611
|
if (anonymous == null){
|
612
|
anonymous = Person.NewTitledInstance("Anon.");
|
613
|
anonymous.setUuid(uuidAnonymous);
|
614
|
getAgentService().save(anonymous);
|
615
|
}
|
616
|
return anonymous;
|
617
|
}
|
618
|
|
619
|
private void makeDetermination(DerivedUnit specimen, SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
620
|
Map<String, String> record, TaxonName taxonName) {
|
621
|
|
622
|
DeterminationEvent determination;
|
623
|
determination = DeterminationEvent.NewInstance(taxonName, specimen);
|
624
|
determination.setPreferredFlag(true);
|
625
|
|
626
|
//determiner/identifier
|
627
|
TeamOrPersonBase<?> determiner = makeDeterminer(state, record, line);
|
628
|
determination.setDeterminer(determiner);
|
629
|
|
630
|
//date
|
631
|
TimePeriod date = makeIdentificationDate(state, record, line);
|
632
|
determination.setTimeperiod(date);
|
633
|
|
634
|
//qualifier
|
635
|
DefinedTerm qualifier = makeDeterminationQualifier(state, record, line);
|
636
|
determination.setModifier(qualifier);
|
637
|
|
638
|
//history
|
639
|
String history = record.get(COL_IDENTIFICATION_HISTORY);
|
640
|
if (history != null){
|
641
|
String label = "Identification History";
|
642
|
String text = label;
|
643
|
ExtensionType detHistoryType = getExtensionType(state, uuidExtTypeIdentificationHistory, label, text, null);
|
644
|
specimen.addExtension(history, detHistoryType);
|
645
|
}
|
646
|
}
|
647
|
|
648
|
private TeamOrPersonBase<?> makeDeterminer(SimpleExcelSpecimenImportState<CONFIG> state,
|
649
|
Map<String, String> record, String line) {
|
650
|
String identifier = record.get(COL_IDENTIFIER);
|
651
|
if (identifier == null){
|
652
|
return null;
|
653
|
}else if (identifier.equals("Anon.")){
|
654
|
return getAnonymous();
|
655
|
}else{
|
656
|
Person person = Person.NewInstance();
|
657
|
person.setTitleCache(identifier, true);
|
658
|
|
659
|
String[] splits = identifier.split("\\.");
|
660
|
int length = splits.length;
|
661
|
if (splits[length - 1].equals("")){
|
662
|
splits[length - 2]= splits[length - 2]+".";
|
663
|
length--;
|
664
|
}
|
665
|
if (splits[length - 1].startsWith("-")){
|
666
|
splits[length - 2]= splits[length - 2]+"." + splits[length - 1];
|
667
|
length--;
|
668
|
}
|
669
|
String familyName = splits[length - 1];
|
670
|
String initials = null;
|
671
|
for (int i= 0; i < length-1;i++){
|
672
|
initials = CdmUtils.concat("", initials, splits[i]+".");
|
673
|
}
|
674
|
person.setFamilyName(familyName);
|
675
|
person.setInitials(initials);
|
676
|
TeamOrPersonBase<?> result = state.getDeduplicationHelper().getExistingAuthor(person, false);
|
677
|
return result;
|
678
|
}
|
679
|
}
|
680
|
|
681
|
private TimePeriod makeIdentificationDate(SimpleExcelSpecimenImportState<CONFIG> state,
|
682
|
Map<String, String> record, String line) {
|
683
|
String strDate = record.get(COL_IDENTIFICATION_DATE);
|
684
|
if (strDate == null || strDate.equals("s.n.")){
|
685
|
return null;
|
686
|
}
|
687
|
String[] splits = strDate.split("/");
|
688
|
String strYear = splits[splits.length-1];
|
689
|
String strMonth = splits.length < 2? null:splits[splits.length-2];
|
690
|
String strDay = splits.length < 3? null:splits[splits.length-3];
|
691
|
|
692
|
Integer year = intFromString(state, strYear, line, COL_IDENTIFICATION_DATE);
|
693
|
Integer month = intFromString(state, strMonth, line, COL_IDENTIFICATION_DATE);
|
694
|
Integer day = intFromString(state, strDay, line, COL_IDENTIFICATION_DATE);
|
695
|
Partial start = TimePeriodParser.makePartialFromDateParts(year, month, day);
|
696
|
return TimePeriod.NewInstance(start);
|
697
|
}
|
698
|
|
699
|
private DefinedTerm makeDeterminationQualifier(SimpleExcelSpecimenImportState<CONFIG> state,
|
700
|
Map<String, String> record, String line) {
|
701
|
String qualifier = record.get(COL_IDENTIFICATION_QUALIFIER);
|
702
|
if (qualifier != null){
|
703
|
try {
|
704
|
return DeterminationModifierParser.parseDeterminationQualifier(qualifier);
|
705
|
} catch (UnknownCdmTypeException e) {
|
706
|
//TODO add to terms
|
707
|
if (qualifier.equals("vel. aff.")){
|
708
|
|
709
|
DefinedTerm velAff = (DefinedTerm)getTermService().find(uuidDetQualVelAff);
|
710
|
if (velAff == null){
|
711
|
velAff = DefinedTerm.NewModifierInstance(qualifier, qualifier, qualifier);
|
712
|
velAff.setUuid(uuidDetQualVelAff);
|
713
|
getTermService().save(velAff);
|
714
|
}
|
715
|
return velAff;
|
716
|
}
|
717
|
state.getResult().addError("Determination qualifier could not be recognized: " + qualifier, null, line);
|
718
|
return null;
|
719
|
}
|
720
|
}else{
|
721
|
return null;
|
722
|
}
|
723
|
}
|
724
|
|
725
|
private Taxon getTaxon(TaxonBase<?> taxonBase) {
|
726
|
if (taxonBase.isInstanceOf(Synonym.class)){
|
727
|
return CdmBase.deproxy(taxonBase, Synonym.class).getAcceptedTaxon();
|
728
|
}else{
|
729
|
return CdmBase.deproxy(taxonBase, Taxon.class);
|
730
|
}
|
731
|
}
|
732
|
|
733
|
private TaxonBase<?> getTaxonByCdmId(SimpleExcelSpecimenImportState<CONFIG> state, String line,
|
734
|
Map<String, String> record, String noStr) {
|
735
|
|
736
|
String strUuidTaxon = record.get(COL_TAXON_UUID);
|
737
|
if (strUuidTaxon != null && ! state.getConfig().isOnlyNonCdmTaxa()){
|
738
|
UUID uuidTaxon;
|
739
|
try {
|
740
|
uuidTaxon = UUID.fromString(strUuidTaxon);
|
741
|
} catch (Exception e) {
|
742
|
state.getResult().addError("Taxon uuid has incorrect format. Taxon could not be loaded. Data not imported.", null, line);
|
743
|
return null;
|
744
|
}
|
745
|
TaxonBase<?> result = getTaxonService().find(uuidTaxon);
|
746
|
if (result == null){
|
747
|
state.getResult().addError("Taxon for uuid "+strUuidTaxon+" could not be found in database. "
|
748
|
+ "Taxon could not be loaded. Data not imported.", null, line);
|
749
|
|
750
|
}
|
751
|
return result;
|
752
|
}else{
|
753
|
return null;
|
754
|
}
|
755
|
}
|
756
|
|
757
|
@Override
|
758
|
protected IdentifiableSource makeOriginalSource(SimpleExcelSpecimenImportState<CONFIG> state) {
|
759
|
return IdentifiableSource.NewDataImportInstance(getValue(state.getOriginalRecord(), COL_VOUCHER_ID), COL_VOUCHER_ID, getSourceCitation(state));
|
760
|
}
|
761
|
|
762
|
protected Reference getSourceCitation(SimpleExcelSpecimenImportState<CONFIG> state) {
|
763
|
Reference source = state.getConfig().getSourceReference();
|
764
|
if (source.getId() == 0){
|
765
|
Reference persisted = getReferenceService().find(source.getUuid());
|
766
|
if (persisted == null){
|
767
|
getReferenceService().saveOrUpdate(source);
|
768
|
}else{
|
769
|
state.getConfig().setSourceReference(persisted);
|
770
|
source = persisted;
|
771
|
}
|
772
|
}
|
773
|
return source;
|
774
|
}
|
775
|
}
|