Project

General

Profile

Download (39.7 KB) Statistics
| Branch: | Revision:
1
// $Id$
2
/**
3
* Copyright (C) 2009 EDIT
4
* European Distributed Institute of Taxonomy 
5
* http://www.e-taxonomy.eu
6
* 
7
* The contents of this file are subject to the Mozilla Public License Version 1.1
8
* See LICENSE.TXT at the top of this package for the full license terms.
9
*/
10
package eu.etaxonomy.cdm.io.pesi.out;
11

    
12
import static java.util.EnumSet.of;
13

    
14
import java.sql.Connection;
15
import java.sql.PreparedStatement;
16
import java.sql.SQLException;
17
import java.sql.Types;
18
import java.util.Arrays;
19
import java.util.HashSet;
20
import java.util.List;
21
import java.util.Map;
22
import java.util.Set;
23

    
24
import org.apache.log4j.Logger;
25
import org.springframework.stereotype.Component;
26
import org.springframework.transaction.TransactionStatus;
27

    
28
import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
29
import eu.etaxonomy.cdm.io.common.DbExportStateBase;
30
import eu.etaxonomy.cdm.io.common.Source;
31
import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;
32
import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
33
import eu.etaxonomy.cdm.io.common.mapping.out.CollectionExportMapping;
34
import eu.etaxonomy.cdm.io.common.mapping.out.DbAnnotationMapper;
35
import eu.etaxonomy.cdm.io.common.mapping.out.DbAreaMapper;
36
import eu.etaxonomy.cdm.io.common.mapping.out.DbConstantMapper;
37
import eu.etaxonomy.cdm.io.common.mapping.out.DbDescriptionElementTaxonMapper;
38
import eu.etaxonomy.cdm.io.common.mapping.out.DbDistributionStatusMapper;
39
import eu.etaxonomy.cdm.io.common.mapping.out.DbExportIgnoreMapper;
40
import eu.etaxonomy.cdm.io.common.mapping.out.DbExportNotYetImplementedMapper;
41
import eu.etaxonomy.cdm.io.common.mapping.out.DbLanguageMapper;
42
import eu.etaxonomy.cdm.io.common.mapping.out.DbObjectMapper;
43
import eu.etaxonomy.cdm.io.common.mapping.out.DbOriginalNameMapper;
44
import eu.etaxonomy.cdm.io.common.mapping.out.DbSimpleFilterMapper;
45
import eu.etaxonomy.cdm.io.common.mapping.out.DbSingleSourceMapper;
46
import eu.etaxonomy.cdm.io.common.mapping.out.DbStringMapper;
47
import eu.etaxonomy.cdm.io.common.mapping.out.DbTextDataMapper;
48
import eu.etaxonomy.cdm.io.common.mapping.out.IdMapper;
49
import eu.etaxonomy.cdm.io.common.mapping.out.MethodMapper;
50
import eu.etaxonomy.cdm.model.common.CdmBase;
51
import eu.etaxonomy.cdm.model.common.Extension;
52
import eu.etaxonomy.cdm.model.common.ExtensionType;
53
import eu.etaxonomy.cdm.model.common.Language;
54
import eu.etaxonomy.cdm.model.common.LanguageString;
55
import eu.etaxonomy.cdm.model.common.Marker;
56
import eu.etaxonomy.cdm.model.common.MarkerType;
57
import eu.etaxonomy.cdm.model.description.CommonTaxonName;
58
import eu.etaxonomy.cdm.model.description.DescriptionBase;
59
import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
60
import eu.etaxonomy.cdm.model.description.Distribution;
61
import eu.etaxonomy.cdm.model.description.Feature;
62
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
63
import eu.etaxonomy.cdm.model.description.TaxonInteraction;
64
import eu.etaxonomy.cdm.model.description.TaxonNameDescription;
65
import eu.etaxonomy.cdm.model.description.TextData;
66
import eu.etaxonomy.cdm.model.location.NamedArea;
67
import eu.etaxonomy.cdm.model.name.NonViralName;
68
import eu.etaxonomy.cdm.model.name.TaxonNameBase;
69
import eu.etaxonomy.cdm.model.taxon.Taxon;
70
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
71
import eu.etaxonomy.cdm.profiler.ProfilerController;
72
/**
73
 * The export class for {@link eu.etaxonomy.cdm.model.description.DescriptionElementBase DescriptionElements}.<p>
74
 * Inserts into DataWarehouse database table <code>Note</code>.<p>
75
 * It is divided into two phases:<ul>
76
 * <li>Phase 1:	Export of DescriptionElements as Notes.
77
 * <li>Phase 2:	Export of TaxonName extensions <code>taxComment</code>, <code>fauComment</code> and <code>fauExtraCodes</code> as Notes.</ul>
78
 * @author e.-m.lee
79
 * @date 23.02.2010
80
 *
81
 */
82
@Component
83
public class PesiDescriptionExport extends PesiExportBase {
84
	private static final Logger logger = Logger.getLogger(PesiDescriptionExport.class);
85
	
86
	private static final Class<? extends CdmBase> standardMethodParameter = DescriptionElementBase.class;
87

    
88
	private static int modCount = 1000;
89
	private static final String dbNoteTableName = "Note";
90
	private static final String dbOccurrenceTableName = "Occurrence";
91
	private static final String dbVernacularTableName = "CommonName";
92
	private static final String dbImageTableName = "Image";
93
	private static final String dbAdditionalSourceTableName = "AdditionalTaxonSource";
94
	private static final String pluralString = "attached infos";
95
	private static final String parentPluralString = "Taxa";
96

    
97
	//decide where to handle them best (configurator, transformer, single method, ...)
98
	private static Set<Integer> excludedNoteCategories = new HashSet<Integer>(Arrays.asList(new Integer[]{250,251,252,253,10,11,13}));
99

    
100
	
101
	//debugging
102
	private static int countDescriptions;
103
	private static int countTaxa;
104
	private static int countDistribution;
105
	private static int countAdditionalSources;
106
	private static int countImages;
107
	private static int countNotes;
108
	
109
	private static int countCommonName;
110
	private static int countOccurrence;
111
	private static int countOthers;
112
	
113
	public PesiDescriptionExport() {
114
		super();
115
	}
116

    
117
	@Override
118
	public Class<? extends CdmBase> getStandardMethodParameter() {
119
		return standardMethodParameter;
120
	}
121

    
122
	@Override
123
	protected boolean doCheck(PesiExportState state) {
124
		boolean result = true;
125
		return result;
126
	}
127

    
128
	@Override
129
	protected void doInvoke(PesiExportState state) {
130
		try {
131
			logger.info("*** Started Making " + pluralString + " ...");
132

    
133
			// Stores whether this invoke was successful or not.
134
			boolean success = true;
135
	
136
			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Note
137
			PesiExportMapping notesMapping = getNotesMapping();
138
			notesMapping.initialize(state);
139

    
140
			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Occurrence
141
			PesiExportMapping occurrenceMapping = getOccurrenceMapping();
142
			occurrenceMapping.initialize(state);
143

    
144
			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
145
			PesiExportMapping addSourceSourceMapping = getAddTaxonSourceSourceMapping();
146
			addSourceSourceMapping.initialize(state);
147
			PesiExportMapping additionalSourceMapping = getAdditionalTaxonSourceMapping();
148
			additionalSourceMapping.initialize(state);
149

    
150
			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
151
			PesiExportMapping vernacularMapping = getVernacularNamesMapping();
152
			vernacularMapping.initialize(state);
153
			
154
			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
155
			PesiExportMapping imageMapping = getImageMapping();
156
			imageMapping.initialize(state);
157
			
158
			// Start transaction
159
			success &= doPhase01(state, notesMapping, occurrenceMapping, addSourceSourceMapping, additionalSourceMapping, vernacularMapping, imageMapping);
160

    
161
			// Start transaction
162
			success &= doPhase01b(state, notesMapping, occurrenceMapping, addSourceSourceMapping, additionalSourceMapping, vernacularMapping, imageMapping);
163

    
164
			
165
			logger.info("PHASE 2...");
166
			success &= doPhase02(state);
167

    
168

    
169
			logger.info("*** Finished Making " + pluralString + " ..." + getSuccessString(success));
170
			
171
			if (!success){
172
				state.setUnsuccessfull();
173
			}
174
			return;
175
		} catch (SQLException e) {
176
			e.printStackTrace();
177
			logger.error(e.getMessage());
178
			state.setUnsuccessfull();
179
		}
180
	}
181

    
182
	//PHASE 01: Description Elements
183
	private boolean doPhase01(PesiExportState state, PesiExportMapping notesMapping, PesiExportMapping occurrenceMapping, PesiExportMapping addSourceSourceMapping, 
184
			PesiExportMapping additionalSourceMapping, PesiExportMapping vernacularMapping, PesiExportMapping imageMapping) throws SQLException {
185
		logger.info("PHASE 1...");
186
		int count = 0;
187
		int pastCount = 0;
188
		boolean success = true;
189
		int limit = state.getConfig().getLimitSave();
190

    
191
		List<Taxon> taxonList = null;
192
		
193
		TransactionStatus txStatus = startTransaction(true);
194
		logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") ...");
195
		List<String> propPath = Arrays.asList(new String[]{"descriptions.elements.*"});
196
		
197
		logger.debug("Start snapshot, before starting loop");
198
		ProfilerController.memorySnapshot();
199
		//taxon descriptions
200
		int partitionCount = 0;
201
		while ((taxonList = getNextTaxonPartition(Taxon.class, limit, partitionCount++, propPath )) != null   ) {
202

    
203
			logger.info("Fetched " + taxonList.size() + " " + pluralString + ". Exporting...");
204
			
205
			for (Taxon taxon : taxonList) {
206
				countTaxa++;
207
				doCount(count++, modCount, pluralString);
208
				state.setCurrentTaxon(taxon);
209
				success &= handleSingleTaxon(taxon, state, notesMapping, occurrenceMapping, addSourceSourceMapping, 
210
						additionalSourceMapping, vernacularMapping, imageMapping);	
211
			}
212
			taxonList = null;
213
			state.setCurrentTaxon(null);
214

    
215
			// Commit transaction
216
			commitTransaction(txStatus);
217
			logger.info("Exported " + (count - pastCount) + " " + pluralString + ". Total: " + count);
218
			pastCount = count;
219

    
220
			// Start transaction
221
			txStatus = startTransaction(true);
222
			logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") for description import ...");
223
		}
224
		
225
	
226
		logger.info("No " + pluralString + " left to fetch.");
227
		logger.info("Partition: " + partitionCount);
228
		logger.info("Taxa: " + countTaxa);
229
		logger.info("Desc: " + countDescriptions);
230
		logger.info("Distr: " + countDistribution);
231
		logger.info("Occur: " + countOccurrence);
232
		logger.info("Commons: " + countCommonName);
233
		logger.info("AddSrc: " + countAdditionalSources);
234
		logger.info("Images: " + countImages);
235
		logger.info("Notes: " + countNotes);
236
		logger.info("Others: " + countOthers);
237
		
238
		// Commit transaction
239
		commitTransaction(txStatus);
240
		logger.debug("Committed transaction.");
241
		return success;
242
	}
243
	
244
	//PHASE 01b: Name Descriptions
245
	private boolean doPhase01b(PesiExportState state, PesiExportMapping notesMapping, PesiExportMapping occurrenceMapping, PesiExportMapping addSourceSourceMapping, 
246
			PesiExportMapping additionalSourceMapping, PesiExportMapping vernacularMapping, PesiExportMapping imageMapping) throws SQLException {
247
		logger.info("PHASE 1b...");
248
		int count = 0;
249
		int pastCount = 0;
250
		boolean success = true;
251
		int limit = state.getConfig().getLimitSave();
252
		
253
		List<TaxonNameDescription> nameDescList = null;
254
		
255
		TransactionStatus txStatus = startTransaction(true);
256
		logger.info("Started new transaction. Fetching some name descriptions (max: " + limit + ") ...");
257
		List<String> propPath = Arrays.asList(new String[]{"descriptions.elements.*"});
258
		
259
		//name descriptions
260
		int partitionCount = 0;
261
		while ((nameDescList = getNextNameDescriptionPartition( limit, partitionCount++, propPath )) != null   ) {
262
			
263
			logger.info("Fetched " + nameDescList.size() + " name descriptions. Exporting...");
264
			
265
			for (TaxonNameDescription desc : nameDescList) {
266
				countTaxa++;
267
				doCount(count++, modCount, "name descriptions");
268
				boolean isImageGallery = desc.isImageGallery();
269
				
270
				TaxonNameBase<?,?> name = desc.getTaxonName();
271
				
272
				for (DescriptionElementBase element : desc.getElements()){
273
					if (isPurePesiName(name)){
274
						success &= handleDescriptionElement(state, notesMapping, occurrenceMapping, vernacularMapping, imageMapping,
275
								addSourceSourceMapping, additionalSourceMapping, isImageGallery, element);
276
					}else{
277
						for (TaxonBase<?> taxonBase : name.getTaxonBases()){
278
							if (isPesiTaxon(taxonBase)){
279
								state.setCurrentTaxon(taxonBase);
280
								success &= handleDescriptionElement(state, notesMapping, occurrenceMapping, vernacularMapping, imageMapping,
281
										addSourceSourceMapping, additionalSourceMapping, isImageGallery, element);
282
								state.setSourceForAdditionalSourceCreated(true);
283
							}
284
						}
285
						state.setSourceForAdditionalSourceCreated(false);
286
					}
287
				}
288
			}
289
			nameDescList = null;
290
			state.setCurrentTaxon(null);
291

    
292
			// Commit transaction
293
			commitTransaction(txStatus);
294
			logger.info("Exported " + (count - pastCount) + " name descriptions. Total: " + count);
295
			pastCount = count;
296

    
297
			// Start transaction
298
			txStatus = startTransaction(true);
299
			logger.info("Started new transaction. Fetching some name descriptions (max: " + limit + ") for description import ...");
300
		}
301
		
302
		logger.info("No " + pluralString + " left to fetch.");
303
		logger.info("Partition: " + partitionCount);
304
		logger.info("Taxa: " + countTaxa);
305
		logger.info("Desc: " + countDescriptions);
306
		logger.info("Distr: " + countDistribution);
307
		logger.info("Occur: " + countOccurrence);
308
		logger.info("Commons: " + countCommonName);
309
		logger.info("AddSrc: " + countAdditionalSources);
310
		logger.info("Images: " + countImages);
311
		logger.info("Notes: " + countNotes);
312
		logger.info("Others: " + countOthers);
313
		
314
		// Commit transaction
315
		commitTransaction(txStatus);
316
		logger.debug("Committed transaction.");
317
		return success;
318
	}
319

    
320
	private boolean handleSingleTaxon(Taxon taxon, PesiExportState state, PesiExportMapping notesMapping, PesiExportMapping occurrenceMapping,
321
			PesiExportMapping addSourceSourceMapping, PesiExportMapping additionalSourceMapping, 
322
			PesiExportMapping vernacularMapping, PesiExportMapping imageMapping) throws SQLException {
323
		boolean success = true;
324
		Set<DescriptionBase<?>> descriptions = new HashSet<DescriptionBase<?>>();
325
		descriptions.addAll(taxon.getDescriptions());
326
		
327
		for (DescriptionBase<?> desc : descriptions){
328
			countDescriptions++;
329

    
330
			boolean isImageGallery = desc.isImageGallery();
331
			for (DescriptionElementBase element : desc.getElements()){
332
				success &= handleDescriptionElement(state, notesMapping, occurrenceMapping, vernacularMapping, imageMapping,
333
						addSourceSourceMapping, additionalSourceMapping, isImageGallery, element);
334
			}
335
		}
336
		descriptions = null;
337
		return success;
338
	}
339

    
340
	private boolean handleDescriptionElement(PesiExportState state, PesiExportMapping notesMapping,
341
			PesiExportMapping occurrenceMapping, PesiExportMapping vernacularMapping, PesiExportMapping imageMapping, 
342
			PesiExportMapping addSourceSourceMapping, PesiExportMapping additionalSourceMapping, boolean isImageGallery, DescriptionElementBase element) throws SQLException {
343
		try {
344
			boolean success = true;
345
			if (isImageGallery){
346
				//TODO handle Images
347
				countImages++;
348
				success &= imageMapping.invoke(element);
349
			}else if (isCommonName(element)){
350
				countCommonName++;
351
				if (element.isInstanceOf(TextData.class)){
352
					//we do not import text data common names
353
				}else{
354
					success &= vernacularMapping.invoke(element);
355
				}
356
			}else if (isOccurrence(element)){
357
				countOccurrence++;
358
				Distribution distribution = CdmBase.deproxy(element, Distribution.class);
359
				MarkerType markerType = getUuidMarkerType(PesiTransformer.uuidMarkerTypeHasNoLastAction, state);
360
				
361
				distribution.addMarker(Marker.NewInstance(markerType, true));
362
				if (isPesiDistribution(state, distribution)){
363
					countDistribution++;
364
					success &=occurrenceMapping.invoke(element);
365
				}
366
			}else if (isAdditionalTaxonSource(element)){
367
				countAdditionalSources++;
368
				if (! state.isSourceForAdditionalSourceCreated()){
369
					success &= addSourceSourceMapping.invoke(element);
370
				}
371
				success &= additionalSourceMapping.invoke(element);
372
			}else if (isExcludedNote(element)){
373
				//do nothing
374
			}else if (isPesiNote(element)){
375
				countNotes++;
376
				success &= notesMapping.invoke(element);
377
			
378
			}else{
379
				countOthers++;
380
				String featureTitle = element.getFeature() == null ? "no feature" :element.getFeature().getTitleCache();
381
				logger.warn("Description element type not yet handled by PESI export: " + element.getUuid() + ", " +  element.getClass() + ", " +  featureTitle);
382
			}
383
			return success;
384
		} catch (Exception e) {
385
			logger.warn("Exception appeared in description element handling: " + e);
386
			e.printStackTrace();
387
			return false;
388
		}
389
	}
390

    
391
	private boolean isExcludedNote(DescriptionElementBase element) {
392
		Integer categoryFk = PesiTransformer.feature2NoteCategoryFk(element.getFeature());
393
		//TODO decide where to handle them best (configurator, transformer, single method, ...)
394
		return (excludedNoteCategories.contains(categoryFk));
395
	}
396

    
397
	private boolean isPesiDistribution(PesiExportState state, Distribution distribution) {
398
		//currently we use the E+M summary status to decide if a distribution should be exported
399
		if (distribution.getStatus() == null){
400
			return false;
401
		}
402
		
403
		//...this may change in future so we keep the following code
404
		Integer key;
405
		//area filter
406
		NamedArea area = distribution.getArea();
407
		if (area == null){
408
			logger.warn("Area is null for distribution " +  distribution.getUuid());
409
			return false;
410
		}else if (area.getUuid().equals(BerlinModelTransformer.euroMedUuid)){
411
			//E+M area only holds endemic status information and therefore is not exported to PESI
412
			return false;
413
		}else if (area.equals(TdwgAreaProvider.getAreaByTdwgAbbreviation("1"))){
414
			//Europe area never holds status information (may probably be deleted in E+M)
415
			return false;
416
//		}else if (area.equals(TdwgArea.getAreaByTdwgAbbreviation("21"))){
417
//			//Macaronesia records should not be exported to PESI
418
//			return false;
419
//		//TODO exclude Russion areas Rs*, and maybe ohters
420
		
421
		} else
422
			try {
423
				if (state.getTransformer().getKeyByNamedArea(area) == null){
424
					String warning = "Area (%s,%s) not available in PESI transformer for taxon %S: ";
425
					TaxonBase<?> taxon =  state.getCurrentTaxon();
426
					warning = String.format(warning, area.getTitleCache(), area.getRepresentation(Language.ENGLISH()).getAbbreviatedLabel(),taxon ==null? "-" : taxon.getTitleCache());
427
					logger.warn(warning);
428
					return false;
429
				}
430
			} catch (UndefinedTransformerMethodException e1) {
431
				logger.warn("Area not available in PESI transformer " +  area.getTitleCache());
432
				return false;
433
			}
434
		return true;
435
		
436
//		
437
//		//status
438
//		PresenceAbsenceTermBase<?> status = distribution.getStatus();
439
//		if (status == null){
440
//			logger.warn("No status for distribution: " +  distribution.getUuid());
441
//			return false;
442
//		}
443
//		try {
444
//			key = (Integer)state.getTransformer().getKeyByPresenceAbsenceTerm(status);
445
//			if (key != null){
446
//				return true;
447
//			}else{
448
//				logger.warn("PresenceAbsenceTerm " + status.getTitleCache() + "not handled in transformer");
449
//				return false;
450
//			}
451
//		} catch (UndefinedTransformerMethodException e) {
452
//			logger.warn("PresenceAbsenceTerm " + status.getTitleCache() + "not handled in transformer");
453
//			return false;
454
//		}
455
	}
456

    
457
	private boolean isPesiNote(DescriptionElementBase element) {
458
		return (getNoteCategoryFk(element) != null);
459
	}
460

    
461
	private boolean isAdditionalTaxonSource(DescriptionElementBase element) {
462
		Feature feature = element.getFeature();
463
		if (feature == null){
464
			return false;
465
		}
466
		return (feature.equals(Feature.CITATION()) || feature.equals(Feature.ADDITIONAL_PUBLICATION()));
467
	}
468

    
469
	private boolean isOccurrence(DescriptionElementBase element) {
470
		Feature feature = element.getFeature();
471
		if (feature != null && feature.equals(Feature.DISTRIBUTION())){
472
			return true;
473
		}else if (element.isInstanceOf(Distribution.class)){
474
			logger.warn("Description element has class 'Distribution' but has no feature 'Distribution'");
475
			return true;
476
		}else{
477
			return false;
478
		}
479
	}
480

    
481
	private boolean isCommonName(DescriptionElementBase element) {
482
		Feature feature = element.getFeature();
483
		if (feature == null){
484
			return false;
485
		}
486
		return (feature.equals(Feature.COMMON_NAME()));
487
	}
488

    
489
	//PHASE 02: Name extensions
490
	private boolean doPhase02(PesiExportState state) {
491
		TransactionStatus txStatus;
492
		boolean success =  true;
493
		
494
		// Get the limit for objects to save within a single transaction.
495
		int limit = state.getConfig().getLimitSave();
496
					
497
		txStatus = startTransaction(true);
498
		ExtensionType taxCommentExtensionType = (ExtensionType)getTermService().find(PesiTransformer.taxCommentUuid);
499
		ExtensionType fauCommentExtensionType = (ExtensionType)getTermService().find(PesiTransformer.fauCommentUuid);
500
		ExtensionType fauExtraCodesExtensionType = (ExtensionType)getTermService().find(PesiTransformer.fauExtraCodesUuid);
501
		List<TaxonNameBase> taxonNameList = null;
502
		
503
		int count = 0;
504
		int pastCount = 0;
505
		Connection connection = state.getConfig().getDestination().getConnection();
506
		logger.info("Started new transaction. Fetching some " + parentPluralString + " first (max: " + limit + ") ...");
507
		logger.warn("TODO handle extensions on taxon level, not name level (");
508
		while ((taxonNameList = getNameService().list(null, limit, count, null, null)).size() > 0) {
509

    
510
			logger.info("Fetched " + taxonNameList.size() + " names. Exporting...");
511
			for (TaxonNameBase<?,?> taxonName : taxonNameList) {
512
				Set<Extension> extensions = taxonName.getExtensions();
513
				for (Extension extension : extensions) {
514
					if (extension.getType().equals(taxCommentExtensionType)) {
515
						String taxComment = extension.getValue();
516
						invokeNotes(taxComment, 
517
								PesiTransformer.getNoteCategoryFk(PesiTransformer.taxCommentUuid), 
518
								PesiTransformer.getNoteCategoryCache(PesiTransformer.taxCommentUuid),
519
								null, null, getTaxonKey(taxonName, state),connection);
520
					} else if (extension.getType().equals(fauCommentExtensionType)) {
521
						String fauComment = extension.getValue();
522
						invokeNotes(fauComment, 
523
								PesiTransformer.getNoteCategoryFk(PesiTransformer.fauCommentUuid), 
524
								PesiTransformer.getNoteCategoryCache(PesiTransformer.fauCommentUuid),
525
								null, null, getTaxonKey(taxonName, state),connection);
526
					} else if (extension.getType().equals(fauExtraCodesExtensionType)) {
527
						String fauExtraCodes = extension.getValue();
528
						invokeNotes(fauExtraCodes, 
529
								PesiTransformer.getNoteCategoryFk(PesiTransformer.fauExtraCodesUuid), 
530
								PesiTransformer.getNoteCategoryCache(PesiTransformer.fauExtraCodesUuid),
531
								null, null, getTaxonKey(taxonName, state),connection);
532
					}
533
				}
534
				
535
				doCount(count++, modCount, pluralString);
536
			}
537

    
538
			// Commit transaction
539
			commitTransaction(txStatus);
540
			logger.debug("Committed transaction.");
541
			logger.info("Exported " + (count - pastCount) + " names. Total: " + count);
542
			pastCount = count;
543

    
544
			// Start transaction
545
			txStatus = startTransaction(true);
546
			logger.info("Started new transaction. Fetching some names first (max: " + limit + ") ...");
547
		}
548
		if (taxonNameList.size() == 0) {
549
			logger.info("No names left to fetch.");
550
		}
551
		taxonNameList = null;
552
		// Commit transaction
553
		commitTransaction(txStatus);
554
		logger.debug("Committed transaction.");
555
		return success;
556
	}
557

    
558
	/**
559
	 * @param taxComment
560
	 * @param noteCategoryFk
561
	 * @param noteCategoryCache
562
	 * @param object
563
	 * @param object2
564
	 */
565
	private void invokeNotes(String note, Integer noteCategoryFk,
566
			String noteCategoryCache, Integer languageFk, String languageCache, 
567
			Integer taxonFk, Connection connection) {
568
		String notesSql = "UPDATE Note SET Note_1 = ?, NoteCategoryFk = ?, NoteCategoryCache = ?, LanguageFk = ?, LanguageCache = ? WHERE TaxonFk = ?"; 
569
		try {
570
			PreparedStatement notesStmt = connection.prepareStatement(notesSql);
571
			
572
			if (note != null) {
573
				notesStmt.setString(1, note);
574
			} else {
575
				notesStmt.setObject(1, null);
576
			}
577
			
578
			if (noteCategoryFk != null) {
579
				notesStmt.setInt(2, noteCategoryFk);
580
			} else {
581
				notesStmt.setObject(2, null);
582
			}
583
			
584
			if (noteCategoryCache != null) {
585
				notesStmt.setString(3, noteCategoryCache);
586
			} else {
587
				notesStmt.setObject(3, null);
588
			}
589
			
590
			if (languageFk != null) {
591
				notesStmt.setInt(4, languageFk);
592
			} else {
593
				notesStmt.setObject(4, null);
594
			}
595
			
596
			if (languageCache != null) {
597
				notesStmt.setString(5, languageCache);
598
			} else {
599
				notesStmt.setObject(5, null);
600
			}
601
			
602
			if (taxonFk != null) {
603
				notesStmt.setInt(6, taxonFk);
604
			} else {
605
				notesStmt.setObject(6, null);
606
			}
607
			
608
			notesStmt.executeUpdate();
609
		} catch (SQLException e) {
610
			logger.error("Note could not be created: " + note);
611
			e.printStackTrace();
612
		}
613

    
614

    
615
	}
616

    
617
	/**
618
	 * Deletes all entries of database tables related to <code>Note</code>.
619
	 * @param state The PesiExportState
620
	 * @return Whether the delete operation was successful or not.
621
	 */
622
	protected boolean doDelete(PesiExportState state) {
623
		PesiExportConfigurator pesiConfig = (PesiExportConfigurator) state.getConfig();
624
		
625
		String sql;
626
		Source destination =  pesiConfig.getDestination();
627

    
628
		// Clear NoteSource
629
		sql = "DELETE FROM NoteSource";
630
		destination.setQuery(sql);
631
		destination.update(sql);
632

    
633
		// Clear Note
634
		sql = "DELETE FROM " + dbNoteTableName;
635
		destination.setQuery(sql);
636
		destination.update(sql);
637
		return true;
638
	}
639

    
640
	/* (non-Javadoc)
641
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IoStateBase)
642
	 */
643
	@Override
644
	protected boolean isIgnore(PesiExportState state) {
645
		return ! state.getConfig().isDoDescription();
646
	}
647

    
648

    
649
	/**
650
	 * Returns the <code>Note_2</code> attribute.
651
	 * @param descriptionElement The {@link DescriptionElementBase DescriptionElement}.
652
	 * @return The <code>Note_2</code> attribute.
653
	 * @see MethodMapper
654
	 */
655
	@SuppressWarnings("unused") //used for mapper
656
	private static String getNote_2(DescriptionElementBase element) {
657
		//E+M map links -> medium
658
		if (element.getFeature() != null && element.getFeature().getUuid().equals(BerlinModelTransformer.uuidFeatureMaps)){
659
			String text = CdmBase.deproxy(element, TextData.class).getText(Language.ENGLISH());
660
			if (text.contains("medium")){
661
				return "medium";
662
			}
663
		}
664
		return null;
665
	}
666

    
667
	/**
668
	 * Returns the <code>NoteCategoryFk</code> attribute.
669
	 * @param descriptionElement The {@link DescriptionElementBase DescriptionElement}.
670
	 * @return The <code>NoteCategoryFk</code> attribute.
671
	 * @see MethodMapper
672
	 */
673
	private static Integer getNoteCategoryFk(DescriptionElementBase descriptionElement) {
674
		Integer result = null;
675
		result = PesiTransformer.feature2NoteCategoryFk(descriptionElement.getFeature());
676
		//TODO decide where to handle them best (configurator, transformer, single method, ...)
677
		if (excludedNoteCategories.contains(result)){
678
			result = null;
679
		}
680
		return result;
681
	}
682
	
683
	/**
684
	 * Returns the <code>NoteCategoryCache</code> attribute.
685
	 * @param descriptionElement The {@link DescriptionElementBase DescriptionElement}.
686
	 * @return The <code>NoteCategoryCache</code> attribute.
687
	 * @see MethodMapper
688
	 */
689
	@SuppressWarnings("unused")
690
	private static String getNoteCategoryCache(DescriptionElementBase descriptionElement, PesiExportState state) {
691
		return state.getTransformer().getCacheByFeature(descriptionElement.getFeature());
692
	}
693
	
694
	
695

    
696

    
697
	/**
698
	 * Returns the <code>LanguageFk</code> attribute.
699
	 * @param descriptionElement The {@link DescriptionElementBase DescriptionElement}.
700
	 * @return The <code>LanguageFk</code> attribute.
701
	 * @see MethodMapper
702
	 */
703
	@SuppressWarnings("unused")
704
	private static Integer getLanguageFk(DescriptionElementBase descriptionElement) {
705
		Language language = getLanguage(descriptionElement);
706

    
707
		return PesiTransformer.language2LanguageId(language);
708
	}
709

    
710
	/**
711
	 * Returns the <code>LanguageCache</code> attribute.
712
	 * @param descriptionElement The {@link DescriptionElementBase DescriptionElement}.
713
	 * @return The <code>LanguageCache</code> attribute.
714
	 * @throws UndefinedTransformerMethodException 
715
	 * @see MethodMapper
716
	 */
717
	@SuppressWarnings("unused")
718
	private static String getLanguageCache(DescriptionElementBase descriptionElement, PesiExportState state) throws UndefinedTransformerMethodException {
719
		Language language = getLanguage(descriptionElement);
720
		return state.getTransformer().getCacheByLanguage(language);
721
	}
722

    
723
	private static Language getLanguage(DescriptionElementBase descriptionElement) {
724
		Language language = null;
725

    
726
		Map<Language, LanguageString> multilanguageText = null;
727
		if (descriptionElement.isInstanceOf(CommonTaxonName.class)) {
728
			CommonTaxonName commonTaxonName = CdmBase.deproxy(descriptionElement, CommonTaxonName.class);
729
			language = commonTaxonName.getLanguage();
730
		} else if (descriptionElement.isInstanceOf(TextData.class)) {
731
			TextData textData = CdmBase.deproxy(descriptionElement, TextData.class);
732
			multilanguageText = textData.getMultilanguageText();
733
		} else if (descriptionElement.isInstanceOf(IndividualsAssociation.class)) {
734
			IndividualsAssociation individualsAssociation = CdmBase.deproxy(descriptionElement, IndividualsAssociation.class);
735
			multilanguageText = individualsAssociation.getDescription();
736
		} else if (descriptionElement.isInstanceOf(TaxonInteraction.class)) {
737
			TaxonInteraction taxonInteraction = CdmBase.deproxy(descriptionElement, TaxonInteraction.class);
738
			multilanguageText = taxonInteraction.getDescriptions();
739
		} else {
740
			logger.debug("Given descriptionElement does not support languages. Hence LanguageCache could not be determined: " + descriptionElement.getUuid());
741
		}
742
		
743
		if (multilanguageText != null) {
744
			Set<Language> languages = multilanguageText.keySet();
745

    
746
			// TODO: Think of something more sophisticated than this
747
			if (languages.size() > 0) {
748
				language = languages.iterator().next();
749
			}
750
			if (languages.size() > 1){
751
				logger.warn("There is more than 1 language for a given description (" + descriptionElement.getClass().getSimpleName() + "):" + descriptionElement.getUuid());
752
			}
753
		}
754
		return language;
755
	}
756

    
757
//	/**
758
//	 * Returns the <code>Region</code> attribute.
759
//	 * @param descriptionElement The {@link DescriptionElementBase DescriptionElement}.
760
//	 * @return The <code>Region</code> attribute.
761
//	 * @see MethodMapper
762
//	 */
763
//	@SuppressWarnings("unused")
764
//	private static String getRegion(DescriptionElementBase descriptionElement) {
765
//		String result = null;
766
//		DescriptionBase<?> inDescription = descriptionElement.getInDescription();
767
//		
768
//		// Area information are associated to TaxonDescriptions and Distributions.
769
//		if (descriptionElement.isInstanceOf(Distribution.class)) {
770
//			Distribution distribution = CdmBase.deproxy(descriptionElement, Distribution.class);
771
//			result = PesiTransformer.area2AreaCache(distribution.getArea());
772
//		} else if (inDescription != null && inDescription.isInstanceOf(TaxonDescription.class)) {
773
//			TaxonDescription taxonDescription = CdmBase.deproxy(inDescription, TaxonDescription.class);
774
//			Set<NamedArea> namedAreas = taxonDescription.getGeoScopes();
775
//			if (namedAreas.size() == 1) {
776
//				result = PesiTransformer.area2AreaCache(namedAreas.iterator().next());
777
//			} else if (namedAreas.size() > 1) {
778
//				logger.warn("This TaxonDescription contains more than one NamedArea: " + taxonDescription.getTitleCache());
779
//			}
780
//		}
781
//		return result;
782
//	}
783

    
784
	
785
	/**
786
	 * Returns the TaxonFk for a given TaxonName or Taxon.
787
	 * @param state The {@link DbExportStateBase DbExportState}.
788
	 * @return
789
	 */
790
	@SuppressWarnings("unused")  //used by mapper
791
	private static Integer getTaxonFk(DescriptionElementBase deb, PesiExportState state) {
792
		TaxonBase<?> entity = state.getCurrentTaxon();
793
		return state.getDbId(entity);
794
	}
795
	
796
	/**
797
	 * Returns the TaxonFk for a given TaxonName.
798
	 * @param taxonName The {@link TaxonNameBase TaxonName}.
799
	 * @param state The {@link DbExportStateBase DbExportState}.
800
	 * @return
801
	 */
802
	private static Integer getTaxonKey(TaxonNameBase<?,?> taxonName, DbExportStateBase<?, PesiTransformer> state) {
803
		return state.getDbId(taxonName);
804
	}
805
	
806
	/**
807
	 * Returns the <code>FullName</code> attribute.
808
	 * @param taxonName The {@link NonViralName NonViralName}.
809
	 * @return The <code>FullName</code> attribute.
810
	 * @see MethodMapper
811
	 */
812
	@SuppressWarnings("unused")
813
	private static String getTaxonFullNameCache(DescriptionElementBase deb, PesiExportState state) {
814
		
815
		TaxonBase<?> taxon =  state.getCurrentTaxon();
816
		TaxonNameBase<?,?> taxonName = taxon.getName();
817
		NonViralName<?> nvn = CdmBase.deproxy(taxonName, NonViralName.class);
818
		String result = getCacheStrategy(nvn).getTitleCache(nvn);
819
		return result;
820
	}
821

    
822

    
823
	/**
824
	 * Returns the CDM to PESI specific export mappings for PESI notes.
825
	 * @return The {@link PesiExportMapping PesiExportMapping}.
826
	 */
827
	private PesiExportMapping getNotesMapping() {
828
		PesiExportMapping mapping = new PesiExportMapping(dbNoteTableName);
829
		
830
		mapping.addMapper(IdMapper.NewInstance("NoteId"));
831
		mapping.addMapper(DbTextDataMapper.NewInstance(Language.ENGLISH(), "Note_1"));
832
		//TODO
833
		mapping.addMapper(MethodMapper.NewInstance("Note_2", this, DescriptionElementBase.class));
834
		mapping.addMapper(MethodMapper.NewInstance("NoteCategoryFk", this, DescriptionElementBase.class ));
835
		
836
		mapping.addMapper(MethodMapper.NewInstance("NoteCategoryCache", this, DescriptionElementBase.class, PesiExportState.class ));
837
		mapping.addMapper(MethodMapper.NewInstance("LanguageFk", this));
838
		mapping.addMapper(MethodMapper.NewInstance("LanguageCache", this, DescriptionElementBase.class, PesiExportState.class));
839
		
840
//		mapping.addMapper(MethodMapper.NewInstance("Region", this));
841
		mapping.addMapper(DbDescriptionElementTaxonMapper.NewInstance("taxonFk"));
842
		mapping.addMapper(ExpertsAndLastActionMapper.NewInstance());
843
		mapping.addCollectionMapping(getNoteSourceMapping());
844
		return mapping;
845
	}
846
	
847
	private CollectionExportMapping<PesiExportState, PesiExportConfigurator,PesiTransformer> getNoteSourceMapping() {
848
		String tableName = "NoteSource";
849
		String collectionAttribute = "sources";
850
		IdMapper parentMapper = IdMapper.NewInstance("NoteFk");
851
		CollectionExportMapping<PesiExportState, PesiExportConfigurator, PesiTransformer> mapping = CollectionExportMapping.NewInstance(tableName, collectionAttribute, parentMapper);
852
		mapping.addMapper(DbSimpleFilterMapper.NewSingleNullAttributeInstance("idInSource", "Sources with idInSource currently handle data lineage"));
853
		mapping.addMapper(DbObjectMapper.NewInstance("Citation", "SourceFk"));
854
		mapping.addMapper(DbObjectMapper.NewInstance("Citation", "SourceNameCache", IS_CACHE));
855
		mapping.addMapper(DbStringMapper.NewInstance("CitationMicroReference", "SourceDetail"));
856
		return mapping;
857
	}
858
	
859

    
860
	/**
861
	 * Returns the CDM to PESI specific export mappings for occurrences.
862
	 * @return The {@link PesiExportMapping PesiExportMapping}.
863
	 */
864
	private PesiExportMapping getOccurrenceMapping() {
865
		PesiExportMapping mapping = new PesiExportMapping(dbOccurrenceTableName);
866
		
867
		mapping.addMapper(IdMapper.NewInstance("OccurrenceId"));
868
		mapping.addMapper(DbDescriptionElementTaxonMapper.NewInstance("taxonFk"));
869
		mapping.addMapper(DbDescriptionElementTaxonMapper.NewInstance("TaxonFullNameCache", true, true, null)); 
870
		
871
		mapping.addMapper(DbAreaMapper.NewInstance(Distribution.class, "Area", "AreaFk", ! IS_CACHE));
872
		mapping.addMapper(DbAreaMapper.NewInstance(Distribution.class, "Area", "AreaNameCache", IS_CACHE));
873
		mapping.addMapper(DbDistributionStatusMapper.NewInstance("OccurrenceStatusFk", ! IS_CACHE));
874
		mapping.addMapper(DbDistributionStatusMapper.NewInstance("OccurrenceStatusCache", IS_CACHE));
875
		
876
//		Use Occurrence source instead
877
		mapping.addMapper(DbExportIgnoreMapper.NewInstance("SourceFk", "Use OccurrenceSource table for sources instead"));
878
		mapping.addMapper(DbExportIgnoreMapper.NewInstance("SourceNameCache", "Use OccurrenceSource table for sources instead"));
879
		
880
		mapping.addMapper(DbAnnotationMapper.NewInstance(null, "Notes"));
881
		mapping.addMapper(ExpertsAndLastActionMapper.NewInstance());
882
		mapping.addCollectionMapping(getOccurrenceSourceMapping());
883
		
884
		return mapping;
885
	}
886

    
887
	private CollectionExportMapping<PesiExportState, PesiExportConfigurator, PesiTransformer> getOccurrenceSourceMapping() {
888
		String tableName = "OccurrenceSource";
889
		String collectionAttribute = "sources";
890
		IdMapper parentMapper = IdMapper.NewInstance("OccurrenceFk");
891
		CollectionExportMapping<PesiExportState, PesiExportConfigurator, PesiTransformer> mapping = CollectionExportMapping.NewInstance(tableName, collectionAttribute, parentMapper);
892
		mapping.addMapper(DbSimpleFilterMapper.NewSingleNullAttributeInstance("idInSource", "Sources with idInSource currently handle data lineage"));
893
		mapping.addMapper(DbObjectMapper.NewInstance("Citation", "SourceFk"));
894
		mapping.addMapper(DbObjectMapper.NewInstance("Citation", "SourceNameCache", IS_CACHE));
895
		mapping.addMapper(DbOriginalNameMapper.NewInstance("OldTaxonName", IS_CACHE, null));
896

    
897
		return mapping;
898
	}
899
	
900

    
901
	/**
902
	 * Returns the CDM to PESI specific export mappings for additional taxon sources to create a new
903
	 * source for the additional source
904
	 * @see #{@link PesiDescriptionExport#getAdditionalTaxonSourceMapping()}
905
	 * @return The {@link PesiExportMapping PesiExportMapping}.
906
	 */
907
	private PesiExportMapping getAddTaxonSourceSourceMapping() {
908
		PesiExportMapping sourceMapping = new PesiExportMapping(PesiSourceExport.dbTableName);
909
		
910
		sourceMapping.addMapper(IdMapper.NewInstance("SourceId"));
911
		sourceMapping.addMapper(DbConstantMapper.NewInstance("SourceCategoryFk", Types.INTEGER, PesiTransformer.REF_UNRESOLVED));
912
		sourceMapping.addMapper(DbConstantMapper.NewInstance("SourceCategoryCache", Types.VARCHAR, PesiTransformer.REF_STR_UNRESOLVED));
913
		
914
//		sourceMapping.addMapper(MethodMapper.NewInstance("NomRefCache", PesiSourceExport.class, "getNomRefCache", Reference.class));
915
		
916
		sourceMapping.addMapper(DbTextDataMapper.NewInstance(Language.ENGLISH(), "NomRefCache"));
917
		
918
		return sourceMapping;
919
	}
920

    
921
	
922
	/**
923
	 * Returns the CDM to PESI specific export mappings for additional taxon sources.
924
	 * @see #{@link PesiDescriptionExport#getAddTaxonSourceSourceMapping()}
925
	 * @return The {@link PesiExportMapping PesiExportMapping}.
926
	 */
927
	private PesiExportMapping getAdditionalTaxonSourceMapping() {
928
	
929
		PesiExportMapping mapping = new PesiExportMapping(dbAdditionalSourceTableName);
930
		
931
		mapping.addMapper(MethodMapper.NewInstance("TaxonFk", this, DescriptionElementBase.class, PesiExportState.class));
932
		
933
		mapping.addMapper(IdMapper.NewInstance("SourceFk"));
934
		mapping.addMapper(DbTextDataMapper.NewInstance(Language.ENGLISH(), "SourceNameCache"));
935
		
936
		mapping.addMapper(DbConstantMapper.NewInstance("SourceUseFk", Types.INTEGER, PesiTransformer.NOMENCLATURAL_REFERENCE));
937
		mapping.addMapper(DbConstantMapper.NewInstance("SourceUseCache", Types.VARCHAR, PesiTransformer.STR_NOMENCLATURAL_REFERENCE));
938
		
939
		mapping.addMapper(DbExportIgnoreMapper.NewInstance("SourceDetail", "SourceDetails not available for additional sources"));
940
		
941
		return mapping;
942
	}
943
	
944
	/**
945
	 * Returns the CDM to PESI specific export mappings for common names.
946
	 * @return The {@link PesiExportMapping PesiExportMapping}.
947
	 */
948
	private PesiExportMapping getVernacularNamesMapping() {
949
		PesiExportMapping mapping = new PesiExportMapping(dbVernacularTableName);
950
		
951
		mapping.addMapper(IdMapper.NewInstance("CommonNameId"));
952
		mapping.addMapper(DbDescriptionElementTaxonMapper.NewInstance("taxonFk"));
953
		
954
		mapping.addMapper(DbStringMapper.NewInstance("Name", "CommonName"));
955
		mapping.addMapper(DbAreaMapper.NewInstance(CommonTaxonName.class, "Area", "Region", IS_CACHE));
956
		
957
		mapping.addMapper(DbLanguageMapper.NewInstance(CommonTaxonName.class, "Language", "LanguageFk", ! IS_CACHE));
958
		mapping.addMapper(DbLanguageMapper.NewInstance(CommonTaxonName.class, "Language", "LanguageCache", IS_CACHE));
959
		
960
		mapping.addMapper(DbSingleSourceMapper.NewInstance("SourceFk", of ( DbSingleSourceMapper.EXCLUDE.WITH_ID) , ! IS_CACHE));
961
		mapping.addMapper(DbSingleSourceMapper.NewInstance("SourceNameCache", of ( DbSingleSourceMapper.EXCLUDE.WITH_ID) , IS_CACHE));
962
		mapping.addMapper(ExpertsAndLastActionMapper.NewInstance());
963
		return mapping;
964

    
965
	}
966
	
967
	private PesiExportMapping getImageMapping() {
968
		PesiExportMapping mapping = new PesiExportMapping(dbImageTableName);
969
		mapping.addMapper(DbDescriptionElementTaxonMapper.NewInstance("taxonFk"));
970
		
971
		//TODO xxx
972
		
973
		return mapping;
974
	}
975

    
976
}
(3-3/12)