1
|
/**
|
2
|
* Copyright (C) 2007 EDIT
|
3
|
* European Distributed Institute of Taxonomy
|
4
|
* http://www.e-taxonomy.eu
|
5
|
*
|
6
|
* The contents of this file are subject to the Mozilla Public License Version 1.1
|
7
|
* See LICENSE.TXT at the top of this package for the full license terms.
|
8
|
*/
|
9
|
|
10
|
package eu.etaxonomy.cdm.io.berlinModel.in;
|
11
|
|
12
|
import java.net.URI;
|
13
|
import java.sql.ResultSet;
|
14
|
import java.sql.SQLException;
|
15
|
import java.util.ArrayList;
|
16
|
import java.util.HashMap;
|
17
|
import java.util.HashSet;
|
18
|
import java.util.List;
|
19
|
import java.util.Map;
|
20
|
import java.util.Set;
|
21
|
import java.util.UUID;
|
22
|
|
23
|
import org.apache.commons.lang.StringUtils;
|
24
|
import org.apache.log4j.Logger;
|
25
|
import org.springframework.stereotype.Component;
|
26
|
import org.springframework.transaction.TransactionStatus;
|
27
|
|
28
|
import eu.etaxonomy.cdm.common.CdmUtils;
|
29
|
import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
|
30
|
import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
|
31
|
import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelOccurrenceImportValidator;
|
32
|
import eu.etaxonomy.cdm.io.common.CdmImportBase;
|
33
|
import eu.etaxonomy.cdm.io.common.IOValidator;
|
34
|
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
|
35
|
import eu.etaxonomy.cdm.io.common.Source;
|
36
|
import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;
|
37
|
import eu.etaxonomy.cdm.model.common.Annotation;
|
38
|
import eu.etaxonomy.cdm.model.common.AnnotationType;
|
39
|
import eu.etaxonomy.cdm.model.common.CdmBase;
|
40
|
import eu.etaxonomy.cdm.model.common.ExtensionType;
|
41
|
import eu.etaxonomy.cdm.model.common.Language;
|
42
|
import eu.etaxonomy.cdm.model.common.Marker;
|
43
|
import eu.etaxonomy.cdm.model.common.MarkerType;
|
44
|
import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
|
45
|
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
|
46
|
import eu.etaxonomy.cdm.model.common.TermType;
|
47
|
import eu.etaxonomy.cdm.model.common.TermVocabulary;
|
48
|
import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
|
49
|
import eu.etaxonomy.cdm.model.description.Distribution;
|
50
|
import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
|
51
|
import eu.etaxonomy.cdm.model.description.TaxonDescription;
|
52
|
import eu.etaxonomy.cdm.model.location.NamedArea;
|
53
|
import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
|
54
|
import eu.etaxonomy.cdm.model.location.NamedAreaType;
|
55
|
import eu.etaxonomy.cdm.model.reference.Reference;
|
56
|
import eu.etaxonomy.cdm.model.taxon.Taxon;
|
57
|
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
|
58
|
import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
|
59
|
|
60
|
|
61
|
/**
|
62
|
* @author a.mueller
|
63
|
* @created 20.03.2008
|
64
|
*/
|
65
|
@Component
|
66
|
public class BerlinModelOccurrenceImport extends BerlinModelImportBase {
|
67
|
private static final Logger logger = Logger.getLogger(BerlinModelOccurrenceImport.class);
|
68
|
|
69
|
public static final String NAMESPACE = "Occurrence";
|
70
|
private static final String EM_AREA_NAMESPACE = "emArea";
|
71
|
|
72
|
private static int modCount = 5000;
|
73
|
private static final String pluralString = "occurrences";
|
74
|
private static final String dbTableName = "emOccurrence"; //??
|
75
|
|
76
|
public BerlinModelOccurrenceImport(){
|
77
|
super(dbTableName, pluralString);
|
78
|
}
|
79
|
|
80
|
@Override
|
81
|
protected String getIdQuery(BerlinModelImportState state) {
|
82
|
String result = " SELECT occurrenceId FROM " + getTableName();
|
83
|
if (StringUtils.isNotBlank(state.getConfig().getOccurrenceFilter())){
|
84
|
result += " WHERE " + state.getConfig().getOccurrenceFilter();
|
85
|
}
|
86
|
return result;
|
87
|
}
|
88
|
|
89
|
@Override
|
90
|
protected String getRecordQuery(BerlinModelImportConfigurator config) {
|
91
|
String emCode = config.isIncludesAreaEmCode()? ", emArea.EMCode" : "";
|
92
|
String strQuery = //DISTINCT because otherwise emOccurrenceSource creates multiple records for a single distribution
|
93
|
" SELECT DISTINCT PTaxon.RIdentifier AS taxonId, emOccurrence.OccurrenceId, emOccurrence.Native, emOccurrence.Introduced, " +
|
94
|
" emOccurrence.Cultivated, emOccurrence.Notes occNotes, " +
|
95
|
" emOccurSumCat.emOccurSumCatId, emOccurSumCat.Short, emOccurSumCat.Description, " +
|
96
|
" emOccurSumCat.OutputCode, emArea.AreaId, emArea.TDWGCode " + emCode +
|
97
|
" FROM emOccurrence INNER JOIN " +
|
98
|
" emArea ON emOccurrence.AreaFk = emArea.AreaId INNER JOIN " +
|
99
|
" PTaxon ON emOccurrence.PTNameFk = PTaxon.PTNameFk AND emOccurrence.PTRefFk = PTaxon.PTRefFk LEFT OUTER JOIN " +
|
100
|
" emOccurSumCat ON emOccurrence.SummaryStatus = emOccurSumCat.emOccurSumCatId LEFT OUTER JOIN " +
|
101
|
" emOccurrenceSource ON emOccurrence.OccurrenceId = emOccurrenceSource.OccurrenceFk " +
|
102
|
" WHERE (emOccurrence.OccurrenceId IN (" + ID_LIST_TOKEN + ") )" +
|
103
|
" ORDER BY PTaxon.RIdentifier";
|
104
|
return strQuery;
|
105
|
}
|
106
|
|
107
|
private Map<Integer, NamedArea> euroMedAreas = new HashMap<Integer, NamedArea>();
|
108
|
|
109
|
|
110
|
@Override
|
111
|
public void doInvoke(BerlinModelImportState state) {
|
112
|
if (state.getConfig().isUseEmAreaVocabulary()){
|
113
|
try {
|
114
|
createEuroMedAreas(state);
|
115
|
} catch (Exception e) {
|
116
|
logger.error("Exception occurred when trying to create euroMed Areas");
|
117
|
e.printStackTrace();
|
118
|
state.setSuccess(false);
|
119
|
}
|
120
|
}
|
121
|
super.doInvoke(state);
|
122
|
//reset
|
123
|
euroMedAreas = new HashMap<Integer, NamedArea>();
|
124
|
}
|
125
|
|
126
|
private TermVocabulary<NamedArea> createEuroMedAreas(BerlinModelImportState state) throws SQLException {
|
127
|
logger.warn("Start creating E+M areas");
|
128
|
Source source = state.getConfig().getSource();
|
129
|
Reference sourceReference = state.getConfig().getSourceReference();
|
130
|
|
131
|
TransactionStatus txStatus = this.startTransaction();
|
132
|
|
133
|
sourceReference = getSourceReference(sourceReference);
|
134
|
|
135
|
TermVocabulary<NamedArea> euroMedAreas = makeEmptyEuroMedVocabulary();
|
136
|
|
137
|
MarkerType eurMarkerType = getMarkerType(state, BerlinModelTransformer.uuidEurArea, "eur", "eur Area", "eur");
|
138
|
MarkerType euroMedAreaMarkerType = getMarkerType(state, BerlinModelTransformer.uuidEurMedArea, "EuroMedArea", "EuroMedArea", "EuroMedArea");
|
139
|
ExtensionType isoCodeExtType = getExtensionType(state, BerlinModelTransformer.uuidIsoCode, "IsoCode", "IsoCode", "iso");
|
140
|
ExtensionType tdwgCodeExtType = getExtensionType(state, BerlinModelTransformer.uuidTdwgAreaCode, "TDWG code", "TDWG Area code", "tdwg");
|
141
|
ExtensionType mclCodeExtType = getExtensionType(state, BerlinModelTransformer.uuidMclCode, "MCL code", "MedCheckList code", "mcl");
|
142
|
NamedAreaLevel areaLevelTop = getNamedAreaLevel(state, BerlinModelTransformer.uuidEuroMedAreaLevelTop, "Euro+Med top area level", "Euro+Med top area level. This level is only to be used for the area representing the complete Euro+Med area", "e+m top", null);
|
143
|
NamedAreaLevel areaLevelEm1 = getNamedAreaLevel(state, BerlinModelTransformer.uuidEuroMedAreaLevelFirst, "Euro+Med 1. area level", "Euro+Med 1. area level", "e+m 1.", null);
|
144
|
NamedAreaLevel areaLevelEm2 = getNamedAreaLevel(state, BerlinModelTransformer.uuidEuroMedAreaLevelSecond, "Euro+Med 2. area level", "Euro+Med 2. area level", "Euro+Med 1. area level", null);
|
145
|
|
146
|
|
147
|
String sql = "SELECT * , CASE WHEN EMCode = 'EM' THEN 'a' ELSE 'b' END as isEM " +
|
148
|
" FROM emArea " +
|
149
|
" ORDER BY isEM, EMCode";
|
150
|
ResultSet rs = source.getResultSet(sql);
|
151
|
|
152
|
NamedArea euroMedArea = null;
|
153
|
NamedArea lastLevel1Area = null;
|
154
|
|
155
|
//euroMedArea (EMCode = 'EM')
|
156
|
rs.next();
|
157
|
euroMedArea = makeSingleEuroMedArea(rs, eurMarkerType, euroMedAreaMarkerType, isoCodeExtType, tdwgCodeExtType, mclCodeExtType,
|
158
|
areaLevelTop, areaLevelEm1 , areaLevelEm2, sourceReference, euroMedArea, lastLevel1Area);
|
159
|
euroMedAreas.addTerm(euroMedArea);
|
160
|
|
161
|
//all other areas
|
162
|
while (rs.next()){
|
163
|
NamedArea newArea = makeSingleEuroMedArea(rs, eurMarkerType, euroMedAreaMarkerType,
|
164
|
isoCodeExtType, tdwgCodeExtType, mclCodeExtType,
|
165
|
areaLevelTop, areaLevelEm1 , areaLevelEm2, sourceReference, euroMedArea, lastLevel1Area);
|
166
|
if (newArea != null){
|
167
|
euroMedAreas.addTerm(newArea);
|
168
|
if (newArea.getPartOf().equals(euroMedArea)){
|
169
|
lastLevel1Area = newArea;
|
170
|
}
|
171
|
}
|
172
|
}
|
173
|
emAreaFinetuning(euroMedAreas, areaLevelEm2);
|
174
|
|
175
|
markAreasAsHidden(state, euroMedAreas);
|
176
|
|
177
|
getVocabularyService().saveOrUpdate(euroMedAreas);
|
178
|
|
179
|
try {
|
180
|
commitTransaction(txStatus);
|
181
|
} catch (Exception e) {
|
182
|
e.printStackTrace();
|
183
|
logger.error("An exception occurred when trying to commit E+M Areas");
|
184
|
}
|
185
|
logger.warn("Created E+M areas");
|
186
|
|
187
|
return euroMedAreas;
|
188
|
}
|
189
|
|
190
|
/**
|
191
|
* @param areaLevelEm2
|
192
|
* @param euroMedAreas2
|
193
|
*/
|
194
|
private void emAreaFinetuning(TermVocabulary<NamedArea> euroMedAreas, NamedAreaLevel areaLevelEm2) {
|
195
|
//CZ
|
196
|
NamedArea oldArea = euroMedAreas.getTermByIdInvocabulary("Cz");
|
197
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Cs"), areaLevelEm2);
|
198
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Sk"), areaLevelEm2);
|
199
|
|
200
|
//Ju
|
201
|
oldArea = euroMedAreas.getTermByIdInvocabulary("Ju");
|
202
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("BH"), areaLevelEm2);
|
203
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Cg"), areaLevelEm2);
|
204
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Ct"), areaLevelEm2);
|
205
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Mk"), areaLevelEm2);
|
206
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Sl"), areaLevelEm2);
|
207
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Sr"), areaLevelEm2);
|
208
|
|
209
|
//IJ
|
210
|
oldArea = euroMedAreas.getTermByIdInvocabulary("IJ");
|
211
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Ir"), areaLevelEm2);
|
212
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Jo"), areaLevelEm2);
|
213
|
|
214
|
//LS
|
215
|
oldArea = euroMedAreas.getTermByIdInvocabulary("LS");
|
216
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Le"), areaLevelEm2);
|
217
|
makeSubterm(oldArea, euroMedAreas.getTermByIdInvocabulary("Sy"), areaLevelEm2);
|
218
|
|
219
|
}
|
220
|
|
221
|
//5.Mark areas to be hidden #3979 .5
|
222
|
private void markAreasAsHidden(BerlinModelImportState state, TermVocabulary<NamedArea> euroMedAreasVoc) {
|
223
|
|
224
|
try {
|
225
|
|
226
|
@SuppressWarnings("unchecked")
|
227
|
TermVocabulary<MarkerType> vocUserDefinedMarkerTypes = getVocabularyService().find(CdmImportBase.uuidUserDefinedMarkerTypeVocabulary);
|
228
|
if (vocUserDefinedMarkerTypes == null){
|
229
|
String message = "Marker type vocabulary could not be found. Hidden areas not added.";
|
230
|
logger.error(message);
|
231
|
System.out.println(message);
|
232
|
}
|
233
|
MarkerType hiddenAreaMarkerType = getMarkerType(state, BerlinModelTransformer.uuidHiddenArea, "Hidden Area","Used to hide distributions for the named areas in publications", null, vocUserDefinedMarkerTypes);
|
234
|
|
235
|
//Add hidden area marker to Rs(C) and Rs(N)
|
236
|
hideArea(euroMedAreasVoc, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs);
|
237
|
hideArea(euroMedAreasVoc, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_B);
|
238
|
hideArea(euroMedAreasVoc, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_C);
|
239
|
hideArea(euroMedAreasVoc, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_E);
|
240
|
hideArea(euroMedAreasVoc, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_N);
|
241
|
hideArea(euroMedAreasVoc, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_K);
|
242
|
hideArea(euroMedAreasVoc, hiddenAreaMarkerType, BerlinModelTransformer.uuidRs_W);
|
243
|
} catch (Exception e) {
|
244
|
e.printStackTrace();
|
245
|
logger.error("Exception in markAreasAsHidden: " + e.getMessage());
|
246
|
}
|
247
|
|
248
|
}
|
249
|
|
250
|
private void hideArea(TermVocabulary<NamedArea> euroMedAreasVoc, MarkerType hiddenAreaMarkerType, UUID areaUuid) {
|
251
|
for (NamedArea namedArea : euroMedAreasVoc){
|
252
|
if (namedArea.getUuid().equals(areaUuid)){
|
253
|
namedArea.addMarker(Marker.NewInstance(hiddenAreaMarkerType, true));
|
254
|
return;
|
255
|
}
|
256
|
}
|
257
|
}
|
258
|
|
259
|
/**
|
260
|
* @param oldArea
|
261
|
* @param namedArea
|
262
|
* @param areaLevelEm2
|
263
|
*/
|
264
|
private void makeSubterm(NamedArea oldArea, NamedArea namedArea, NamedAreaLevel areaLevelEm2) {
|
265
|
namedArea.setLevel(areaLevelEm2);
|
266
|
namedArea.setPartOf(oldArea);
|
267
|
}
|
268
|
|
269
|
/**
|
270
|
* @param sourceReference
|
271
|
* @return
|
272
|
*/
|
273
|
private Reference getSourceReference(Reference sourceReference) {
|
274
|
Reference persistentSourceReference = getReferenceService().find(sourceReference.getUuid()); //just to be sure
|
275
|
if (persistentSourceReference != null){
|
276
|
sourceReference = persistentSourceReference;
|
277
|
}
|
278
|
return sourceReference;
|
279
|
}
|
280
|
|
281
|
/**
|
282
|
* @param eurMarkerType
|
283
|
* @param euroMedAreaMarkerType
|
284
|
* @param isoCodeExtType
|
285
|
* @param tdwgCodeExtType
|
286
|
* @param mclCodeExtType
|
287
|
* @param rs
|
288
|
* @param areaLevelEm2
|
289
|
* @param areaLevelEm1
|
290
|
* @param areaLevelTop
|
291
|
* @throws SQLException
|
292
|
*/
|
293
|
private NamedArea makeSingleEuroMedArea(ResultSet rs, MarkerType eurMarkerType,
|
294
|
MarkerType euroMedAreaMarkerType, ExtensionType isoCodeExtType,
|
295
|
ExtensionType tdwgCodeExtType, ExtensionType mclCodeExtType,
|
296
|
NamedAreaLevel areaLevelTop, NamedAreaLevel areaLevelEm1, NamedAreaLevel areaLevelEm2,
|
297
|
Reference sourceReference, NamedArea euroMedArea, NamedArea level1Area) throws SQLException {
|
298
|
Integer areaId = rs.getInt("AreaId");
|
299
|
String emCode = nullSafeTrim(rs.getString("EMCode"));
|
300
|
String isoCode = nullSafeTrim(rs.getString("ISOCode"));
|
301
|
String tdwgCode = nullSafeTrim(rs.getString("TDWGCode"));
|
302
|
String unit = nullSafeTrim(rs.getString("Unit"));
|
303
|
// ,[Status]
|
304
|
// ,[OutputOrder]
|
305
|
boolean eurMarker = rs.getBoolean("eur");
|
306
|
boolean euroMedAreaMarker = rs.getBoolean("EuroMedArea");
|
307
|
String notes = nullSafeTrim(rs.getString("Notes"));
|
308
|
String mclCode = nullSafeTrim(rs.getString("MCLCode"));
|
309
|
String geoSearch = nullSafeTrim(rs.getString("NameForGeoSearch"));
|
310
|
|
311
|
|
312
|
|
313
|
if (isBlank(emCode)){
|
314
|
emCode = unit;
|
315
|
}
|
316
|
|
317
|
//uuid
|
318
|
UUID uuid = BerlinModelTransformer.getEMAreaUuid(emCode);
|
319
|
NamedArea area = (NamedArea)getTermService().find(uuid);
|
320
|
if (area == null){
|
321
|
//label
|
322
|
area = NamedArea.NewInstance(geoSearch, unit, emCode);
|
323
|
if (uuid != null){
|
324
|
area.setUuid(uuid);
|
325
|
}else{
|
326
|
if (areaId == 211 || areaId == 213){ //Additional Azores and Canary Is. area are merged into primary area, see also area.addSource part below
|
327
|
return null;
|
328
|
}
|
329
|
logger.warn("Uuid for emCode could not be defined: " + emCode);
|
330
|
}
|
331
|
}
|
332
|
|
333
|
|
334
|
//code
|
335
|
area.setIdInVocabulary(emCode);
|
336
|
//notes
|
337
|
if (StringUtils.isNotEmpty(notes)){
|
338
|
area.addAnnotation(Annotation.NewInstance(notes, AnnotationType.EDITORIAL(), Language.DEFAULT()));
|
339
|
}
|
340
|
//markers
|
341
|
area.addMarker(Marker.NewInstance(eurMarkerType, eurMarker));
|
342
|
area.addMarker(Marker.NewInstance(euroMedAreaMarkerType, euroMedAreaMarker));
|
343
|
|
344
|
//extensions
|
345
|
if (isNotBlank(isoCode)){
|
346
|
area.addExtension(isoCode, isoCodeExtType);
|
347
|
}
|
348
|
if (isNotBlank(tdwgCode)){
|
349
|
area.addExtension(tdwgCode, tdwgCodeExtType);
|
350
|
}
|
351
|
if (isNotBlank(mclCode)){
|
352
|
area.addExtension(mclCode, mclCodeExtType);
|
353
|
}
|
354
|
|
355
|
//type
|
356
|
area.setType(NamedAreaType.ADMINISTRATION_AREA());
|
357
|
|
358
|
//source
|
359
|
area.addSource(OriginalSourceType.Import, String.valueOf(areaId), EM_AREA_NAMESPACE, sourceReference, null);
|
360
|
//add duplicate area ids for canary
|
361
|
if (areaId == 624){ //Canary Is.
|
362
|
area.addSource(OriginalSourceType.Import, String.valueOf(213), EM_AREA_NAMESPACE, sourceReference, null);
|
363
|
}
|
364
|
if (areaId == 210){//Azores
|
365
|
area.addSource(OriginalSourceType.Import, String.valueOf(211), EM_AREA_NAMESPACE, sourceReference, null);
|
366
|
}
|
367
|
|
368
|
//parent
|
369
|
if (euroMedArea != null){
|
370
|
if (emCode.contains("(")){
|
371
|
area.setPartOf(level1Area);
|
372
|
area.setLevel(areaLevelEm2);
|
373
|
}else{
|
374
|
area.setPartOf(euroMedArea);
|
375
|
area.setLevel(areaLevelEm1);
|
376
|
}
|
377
|
}else{
|
378
|
area.setLevel(areaLevelTop);
|
379
|
}
|
380
|
this.euroMedAreas.put(areaId, area);
|
381
|
|
382
|
//save
|
383
|
getTermService().saveOrUpdate(area);
|
384
|
|
385
|
return area;
|
386
|
}
|
387
|
|
388
|
private String nullSafeTrim(String string) {
|
389
|
if (string == null){
|
390
|
return null;
|
391
|
}else{
|
392
|
return string.trim();
|
393
|
}
|
394
|
}
|
395
|
|
396
|
/**
|
397
|
*
|
398
|
*/
|
399
|
private TermVocabulary<NamedArea> makeEmptyEuroMedVocabulary() {
|
400
|
TermType type = TermType.NamedArea;
|
401
|
String description = "Euro+Med area vocabulary";
|
402
|
String label = "E+M areas";
|
403
|
String abbrev = null;
|
404
|
URI termSourceUri = null;
|
405
|
OrderedTermVocabulary<NamedArea> result = OrderedTermVocabulary.NewInstance(type, description, label, abbrev, termSourceUri);
|
406
|
|
407
|
result.setUuid(BerlinModelTransformer.uuidVocEuroMedAreas);
|
408
|
getVocabularyService().save(result);
|
409
|
return result;
|
410
|
}
|
411
|
|
412
|
@Override
|
413
|
public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
|
414
|
boolean success = true;
|
415
|
Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
|
416
|
|
417
|
Map<String, TaxonBase<?>> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
|
418
|
|
419
|
ResultSet rs = partitioner.getResultSet();
|
420
|
|
421
|
try {
|
422
|
//map to store the mapping of duplicate berlin model occurrences to their real distributions
|
423
|
//duplicated may occur due to area mappings from BM areas to TDWG areas
|
424
|
Map<Integer, String> duplicateMap = new HashMap<Integer, String>();
|
425
|
int oldTaxonId = -1;
|
426
|
TaxonDescription oldDescription = null;
|
427
|
int i = 0;
|
428
|
int countDescriptions = 0;
|
429
|
int countDistributions = 0;
|
430
|
int countDuplicates = 0;
|
431
|
//for each reference
|
432
|
while (rs.next()){
|
433
|
|
434
|
if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("Facts handled: " + (i-1));}
|
435
|
|
436
|
int occurrenceId = rs.getInt("OccurrenceId");
|
437
|
int newTaxonId = rs.getInt("taxonId");
|
438
|
String notes = nullSafeTrim(rs.getString("occNotes"));
|
439
|
|
440
|
Integer emStatusId = nullSafeInt(rs, "emOccurSumCatId");
|
441
|
|
442
|
try {
|
443
|
//status
|
444
|
PresenceAbsenceTerm status = null;
|
445
|
String alternativeStatusString = null;
|
446
|
if (emStatusId != null){
|
447
|
status = BerlinModelTransformer.occStatus2PresenceAbsence(emStatusId);
|
448
|
}else{
|
449
|
String[] stringArray = new String[]{rs.getString("Native"), rs.getString("Introduced"), rs.getString("Cultivated")};
|
450
|
alternativeStatusString = CdmUtils.concat(",", stringArray);
|
451
|
}
|
452
|
|
453
|
Reference sourceRef = state.getTransactionalSourceReference();
|
454
|
|
455
|
List<NamedArea> areas = makeAreaList(state, rs, occurrenceId);
|
456
|
|
457
|
//create description(elements)
|
458
|
TaxonDescription taxonDescription = getTaxonDescription(newTaxonId, oldTaxonId, oldDescription, taxonMap, occurrenceId, sourceRef);
|
459
|
for (NamedArea area : areas){
|
460
|
Distribution distribution = Distribution.NewInstance(area, status);
|
461
|
if (status == null){
|
462
|
AnnotationType annotationType = AnnotationType.EDITORIAL();
|
463
|
Annotation annotation = Annotation.NewInstance(alternativeStatusString, annotationType, null);
|
464
|
distribution.addAnnotation(annotation);
|
465
|
distribution.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), false));
|
466
|
}
|
467
|
// distribution.setCitation(sourceRef);
|
468
|
if (taxonDescription != null) {
|
469
|
Distribution duplicate = checkIsNoDuplicate(taxonDescription, distribution, duplicateMap , occurrenceId);
|
470
|
if (duplicate == null){
|
471
|
taxonDescription.addElement(distribution);
|
472
|
distribution.addImportSource(String.valueOf(occurrenceId), NAMESPACE, state.getTransactionalSourceReference(), null);
|
473
|
countDistributions++;
|
474
|
if (taxonDescription != oldDescription){
|
475
|
taxaToSave.add(taxonDescription.getTaxon());
|
476
|
oldDescription = taxonDescription;
|
477
|
countDescriptions++;
|
478
|
}
|
479
|
}else{
|
480
|
countDuplicates++;
|
481
|
duplicate.addImportSource(String.valueOf(occurrenceId), NAMESPACE, state.getTransactionalSourceReference(), null);
|
482
|
logger.info("Distribution is duplicate"); }
|
483
|
} else {
|
484
|
logger.warn("Distribution " + area.getLabel() + " ignored. OccurrenceId = " + occurrenceId);
|
485
|
success = false;
|
486
|
}
|
487
|
//notes
|
488
|
if (isNotBlank(notes)){
|
489
|
Annotation annotation = Annotation.NewInstance(notes, Language.DEFAULT());
|
490
|
distribution.addAnnotation(annotation);
|
491
|
}
|
492
|
}
|
493
|
} catch (UnknownCdmTypeException e) {
|
494
|
logger.error("Unknown presenceAbsence status id: " + emStatusId);
|
495
|
e.printStackTrace();
|
496
|
success = false;
|
497
|
}
|
498
|
}
|
499
|
|
500
|
logger.info("Distributions: " + countDistributions + ", Descriptions: " + countDescriptions );
|
501
|
logger.info("Duplicate occurrences: " + (countDuplicates));
|
502
|
|
503
|
logger.info("Taxa to save: " + taxaToSave.size());
|
504
|
getTaxonService().save(taxaToSave);
|
505
|
|
506
|
return success;
|
507
|
} catch (SQLException e) {
|
508
|
logger.error("SQLException:" + e);
|
509
|
return false;
|
510
|
}
|
511
|
}
|
512
|
|
513
|
/**
|
514
|
* @param state
|
515
|
* @param rs
|
516
|
* @param occurrenceId
|
517
|
* @param tdwgCodeString
|
518
|
* @param emCodeString
|
519
|
* @return
|
520
|
* @throws SQLException
|
521
|
*/
|
522
|
//Create area list
|
523
|
private List<NamedArea> makeAreaList(BerlinModelImportState state, ResultSet rs, int occurrenceId) throws SQLException {
|
524
|
List<NamedArea> areas = new ArrayList<NamedArea>();
|
525
|
|
526
|
if (state.getConfig().isUseEmAreaVocabulary()){
|
527
|
Integer areaId = rs.getInt("AreaId");
|
528
|
NamedArea area = this.euroMedAreas.get(areaId);
|
529
|
areas.add(area);
|
530
|
}else{
|
531
|
String tdwgCodeString = rs.getString("TDWGCode");
|
532
|
String emCodeString = state.getConfig().isIncludesAreaEmCode() ? rs.getString("EMCode") : null;
|
533
|
|
534
|
if (tdwgCodeString != null){
|
535
|
|
536
|
String[] tdwgCodes = new String[]{tdwgCodeString};
|
537
|
if (state.getConfig().isSplitTdwgCodes()){
|
538
|
tdwgCodes = tdwgCodeString.split(";");
|
539
|
}
|
540
|
|
541
|
for (String tdwgCode : tdwgCodes){
|
542
|
NamedArea area = TdwgAreaProvider.getAreaByTdwgAbbreviation(tdwgCode.trim());
|
543
|
if (area == null){
|
544
|
area = getOtherAreas(state, emCodeString, tdwgCodeString);
|
545
|
}
|
546
|
if (area != null){
|
547
|
areas.add(area);
|
548
|
}
|
549
|
}
|
550
|
}
|
551
|
|
552
|
if (areas.size()== 0){
|
553
|
NamedArea area = getOtherAreas(state, emCodeString, tdwgCodeString);
|
554
|
if (area != null){
|
555
|
areas.add(area);
|
556
|
}
|
557
|
}
|
558
|
if (areas.size() == 0){
|
559
|
String areaId = rs.getString("AreaId");
|
560
|
logger.warn("No areas defined for occurrence " + occurrenceId + ". EMCode: " + CdmUtils.Nz(emCodeString).trim() + ". AreaId: " + areaId );
|
561
|
}
|
562
|
}
|
563
|
return areas;
|
564
|
}
|
565
|
|
566
|
@Override
|
567
|
public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
|
568
|
String nameSpace;
|
569
|
Class<?> cdmClass;
|
570
|
Set<String> idSet;
|
571
|
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
|
572
|
|
573
|
try{
|
574
|
Set<String> taxonIdSet = new HashSet<String>();
|
575
|
while (rs.next()){
|
576
|
handleForeignKey(rs, taxonIdSet, "taxonId");
|
577
|
}
|
578
|
|
579
|
//taxon map
|
580
|
nameSpace = BerlinModelTaxonImport.NAMESPACE;
|
581
|
cdmClass = TaxonBase.class;
|
582
|
idSet = taxonIdSet;
|
583
|
Map<String, TaxonBase> objectMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
|
584
|
result.put(nameSpace, objectMap);
|
585
|
|
586
|
} catch (SQLException e) {
|
587
|
throw new RuntimeException(e);
|
588
|
}
|
589
|
return result;
|
590
|
}
|
591
|
|
592
|
|
593
|
|
594
|
/**
|
595
|
* Tests if a distribution with the same tdwgArea and the same status already exists in the description.
|
596
|
* If so the old distribution is returned
|
597
|
* @param description
|
598
|
* @param tdwgArea
|
599
|
* @return false, if dupplicate exists. True otherwise.
|
600
|
*/
|
601
|
private Distribution checkIsNoDuplicate(TaxonDescription description, Distribution distribution, Map<Integer, String> duplicateMap, Integer bmDistributionId){
|
602
|
for (DescriptionElementBase descElBase : description.getElements()){
|
603
|
if (descElBase.isInstanceOf(Distribution.class)){
|
604
|
Distribution oldDistr = HibernateProxyHelper.deproxy(descElBase, Distribution.class);
|
605
|
NamedArea oldArea = oldDistr.getArea();
|
606
|
if (oldArea != null && oldArea.equals(distribution.getArea())){
|
607
|
PresenceAbsenceTerm oldStatus = oldDistr.getStatus();
|
608
|
if (oldStatus != null && oldStatus.equals(distribution.getStatus())){
|
609
|
duplicateMap.put(bmDistributionId, oldDistr.getSources().iterator().next().getIdInSource());
|
610
|
return oldDistr;
|
611
|
}
|
612
|
}
|
613
|
}
|
614
|
}
|
615
|
return null;
|
616
|
}
|
617
|
|
618
|
/**
|
619
|
* Use same TaxonDescription if two records belong to the same taxon
|
620
|
* @param newTaxonId
|
621
|
* @param oldTaxonId
|
622
|
* @param oldDescription
|
623
|
* @param taxonMap
|
624
|
* @return
|
625
|
*/
|
626
|
private TaxonDescription getTaxonDescription(int newTaxonId, int oldTaxonId, TaxonDescription oldDescription, Map<String, TaxonBase<?>> taxonMap, int occurrenceId, Reference sourceSec){
|
627
|
TaxonDescription result = null;
|
628
|
if (oldDescription == null || newTaxonId != oldTaxonId){
|
629
|
TaxonBase<?> taxonBase = taxonMap.get(String.valueOf(newTaxonId));
|
630
|
//TODO for testing
|
631
|
//TaxonBase taxonBase = Taxon.NewInstance(TaxonNameFactory.NewBotanicalInstance(Rank.SPECIES()), null);
|
632
|
Taxon taxon;
|
633
|
if ( taxonBase instanceof Taxon ) {
|
634
|
taxon = (Taxon) taxonBase;
|
635
|
} else if (taxonBase != null) {
|
636
|
logger.warn("TaxonBase for Occurrence " + occurrenceId + " was not of type Taxon but: " + taxonBase.getClass().getSimpleName());
|
637
|
return null;
|
638
|
} else {
|
639
|
logger.warn("TaxonBase for Occurrence " + occurrenceId + " is null.");
|
640
|
return null;
|
641
|
}
|
642
|
Set<TaxonDescription> descriptionSet= taxon.getDescriptions();
|
643
|
if (descriptionSet.size() > 0) {
|
644
|
result = descriptionSet.iterator().next();
|
645
|
}else{
|
646
|
result = TaxonDescription.NewInstance();
|
647
|
result.setTitleCache(sourceSec.getTitleCache(), true);
|
648
|
taxon.addDescription(result);
|
649
|
}
|
650
|
}else{
|
651
|
result = oldDescription;
|
652
|
}
|
653
|
return result;
|
654
|
}
|
655
|
|
656
|
@Override
|
657
|
protected boolean doCheck(BerlinModelImportState state){
|
658
|
IOValidator<BerlinModelImportState> validator = new BerlinModelOccurrenceImportValidator();
|
659
|
return validator.validate(state);
|
660
|
}
|
661
|
|
662
|
|
663
|
@Override
|
664
|
protected boolean isIgnore(BerlinModelImportState state){
|
665
|
if (! state.getConfig().isDoOccurrence()){
|
666
|
return true;
|
667
|
}else{
|
668
|
if (!this.checkSqlServerColumnExists(state.getConfig().getSource(), "emOccurrence", "OccurrenceId")){
|
669
|
logger.error("emOccurrence table or emOccurrenceId does not exist. Must ignore occurrence import");
|
670
|
return true;
|
671
|
}else{
|
672
|
return false;
|
673
|
}
|
674
|
}
|
675
|
}
|
676
|
|
677
|
}
|