Project

General

Profile

« Previous | Next » 

Revision 892efc69

Added by Andreas Kohlbecker almost 14 years ago

merging /branches/cdmlib/SPRINT-Chichorieae1/ to trunk

View differences:

cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRefImport.java
14 14
import java.util.Collection;
15 15
import java.util.HashMap;
16 16
import java.util.HashSet;
17
import java.util.Iterator;
17 18
import java.util.List;
18 19
import java.util.Map;
19 20
import java.util.Set;
......
30 31
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
31 32
import eu.etaxonomy.cdm.model.common.CdmBase;
32 33
import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
34
import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
33 35
import eu.etaxonomy.cdm.model.description.Feature;
34 36
import eu.etaxonomy.cdm.model.description.TaxonDescription;
35 37
import eu.etaxonomy.cdm.model.description.TextData;
......
40 42
import eu.etaxonomy.cdm.model.taxon.Taxon;
41 43
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
42 44

  
45
import java.util.Map.Entry;
46

  
43 47

  
44 48
/**
45 49
 * @author a.babadshanjan
......
78 82
//		return false;
79 83
//		}
80 84
	}
81

  
85
	
82 86
	/* (non-Javadoc)
83 87
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IImportConfigurator, eu.etaxonomy.cdm.api.application.CdmApplicationController, java.util.Map)
84 88
	 */
......
87 91

  
88 92
		TransactionStatus txStatus = null;
89 93
		List<TaxonBase> taxonList = null;
94
		List<ReferenceBase> referenceList = null;
90 95
		Set<UUID> taxonUuids = null;
91
		Set<ReferenceBase> references = null;
92
		Set<TeamOrPersonBase> authors = null;
96
		Map<Integer, ReferenceBase> references = null;
97
		Map<String,TeamOrPersonBase> authors = null;
93 98
		Map<UUID, FaunaEuropaeaReferenceTaxon> fauEuTaxonMap = null;
99
		Map<Integer, UUID> referenceUuids = new HashMap<Integer, UUID>();
100
		Set<Integer> referenceIDs = null;
94 101
		int limit = state.getConfig().getLimitSave();
95 102

  
96 103
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
......
100 107
		boolean success = true;
101 108
		int i = 0;
102 109

  
103
		String selectCount = 
110
		String selectCountTaxRefs = 
104 111
			" SELECT count(*) ";
105 112

  
106
		String selectColumns = 
113
		String selectColumnsTaxRefs = 
107 114
			" SELECT Reference.*, TaxRefs.*, Taxon.UUID  ";
108 115
			
109
		String fromClause = 
116
		String fromClauseTaxRefs = 
110 117
			" FROM TaxRefs " +
111 118
			" INNER JOIN Reference ON Reference.ref_id = TaxRefs.trf_ref_id " +
112 119
			" INNER JOIN Taxon ON TaxRefs.trf_tax_id = Taxon.TAX_ID ";
113 120
		
114
		String orderClause = 
121
		String orderClauseTaxRefs = 
115 122
			" ORDER BY TaxRefs.trf_tax_id";
123
		
124
		String selectCountRefs = 
125
			" SELECT count(*) FROM Reference";
126

  
127
		String selectColumnsRefs = 
128
			" SELECT * FROM Reference order by ref_author";
129
		
116 130
			
117
		String countQuery = 
118
			selectCount + fromClause;
131
		String countQueryTaxRefs = 
132
			selectCountTaxRefs + fromClauseTaxRefs;
119 133

  
120
		String selectQuery = 
121
			selectColumns + fromClause + orderClause;
134
		String selectQueryTaxRefs = 
135
			selectColumnsTaxRefs + fromClauseTaxRefs + orderClauseTaxRefs;
136
		
137
		String countQueryRefs = 
138
			selectCountRefs;
122 139

  
140
		String selectQueryRefs = 
141
			selectColumnsRefs;
142
		
143
		int count;
123 144
		if(logger.isInfoEnabled()) { logger.info("Start making References..."); }
124

  
145
//first add all References to CDM
125 146
		try {
126
			ResultSet rs = source.getResultSet(countQuery);
127
			rs.next();
128
			int count = rs.getInt(1);
147
			ResultSet rsRefs = source.getResultSet(countQueryRefs);
148
			rsRefs.next();
149
			count = rsRefs.getInt(1);
129 150
			
130
			rs = source.getResultSet(selectQuery);
151
			rsRefs = source.getResultSet(selectQueryRefs);
131 152
            								
132 153
	        if (logger.isInfoEnabled()) {
154
	        	logger.info("Get all References..."); 
133 155
				logger.info("Number of rows: " + count);
134
				logger.info("Count Query: " + countQuery);
135
				logger.info("Select Query: " + selectQuery);
156
				logger.info("Count Query: " + countQueryRefs);
157
				logger.info("Select Query: " + selectQueryRefs);
136 158
			}
137

  
138
			while (rs.next()) {
139

  
159
	        
160
	        while (rsRefs.next()){
161
	        	int refId = rsRefs.getInt("ref_id");
162
				String refAuthor = rsRefs.getString("ref_author");
163
				String year = rsRefs.getString("ref_year");
164
				String title = rsRefs.getString("ref_title");
165
				
166
				if (year == null){
167
					try{		
168
						year = String.valueOf((Integer.parseInt(title)));
169
					}
170
					catch(Exception ex)   
171
					{
172
						logger.info("year is empty and " +title + " contains no integer");
173
				    }
174
				}
175
				String refSource = rsRefs.getString("ref_source");
176
	        	
140 177
				if ((i++ % limit) == 0) {
141 178

  
142 179
					txStatus = startTransaction();
143
					taxonUuids = new HashSet<UUID>(limit);
144
					references = new HashSet<ReferenceBase>(limit);
145
					authors = new HashSet<TeamOrPersonBase>(limit);
146
					fauEuTaxonMap = new HashMap<UUID, FaunaEuropaeaReferenceTaxon>(limit);
147

  
180
					references = new HashMap<Integer,ReferenceBase>(limit);
181
					authors = new HashMap<String,TeamOrPersonBase>(limit);
182
					
148 183
					if(logger.isInfoEnabled()) {
149 184
						logger.info("i = " + i + " - Reference import transaction started"); 
150 185
					}
151 186
				}
152

  
153
				int taxonId = rs.getInt("trf_tax_id");
154
				int refId = rs.getInt("ref_id");
155
				String refAuthor = rs.getString("ref_author");
156
				String year = rs.getString("ref_year");
157
				String title = rs.getString("ref_title");
158
				String refSource = rs.getString("ref_source");
159
				String page = rs.getString("trf_page");
160
				UUID currentTaxonUuid = null;
161
				if (resultSetHasColumn(rs, "UUID")){
162
					currentTaxonUuid = UUID.fromString(rs.getString("UUID"));
163
				} else {
164
					logger.error("Taxon (" + taxonId + ") without UUID ignored");
165
					continue;
166
				}
167

  
168
				FaunaEuropaeaReference fauEuReference = new FaunaEuropaeaReference();
169
				fauEuReference.setTaxonUuid(currentTaxonUuid);
170
				fauEuReference.setReferenceId(refId);
171
				fauEuReference.setReferenceAuthor(refAuthor);
172
				fauEuReference.setReferenceYear(year);
173
				fauEuReference.setReferenceTitle(title);
174
				fauEuReference.setReferenceSource(refSource);
175
				fauEuReference.setPage(page);
176

  
177
				if (!taxonUuids.contains(currentTaxonUuid)) {
178
					taxonUuids.add(currentTaxonUuid);
179
					FaunaEuropaeaReferenceTaxon fauEuReferenceTaxon = 
180
						new FaunaEuropaeaReferenceTaxon(currentTaxonUuid);
181
					fauEuTaxonMap.put(currentTaxonUuid, fauEuReferenceTaxon);
182
				} else {
183
					if (logger.isTraceEnabled()) { 
184
						logger.trace("Taxon (" + currentTaxonUuid + ") already stored.");
185
						continue;
186
					}
187
				}
188

  
187
				
189 188
				ReferenceBase<?> reference = null;
190 189
				TeamOrPersonBase<Team> author = null;
191
				ReferenceFactory refFactory = ReferenceFactory.newInstance();
192
				reference = refFactory.newGeneric();
190
				//ReferenceFactory refFactory = ReferenceFactory.newInstance();
191
				reference = ReferenceFactory.newGeneric();
192

  
193 193
				reference.setTitleCache(title);
194 194
				reference.setDatePublished(ImportHelper.getDatePublished(year));
195
				author = Team.NewInstance();
196
				author.setTitleCache(refAuthor);
195
				
196
				if (!authors.containsKey(refAuthor)) {
197
					if (refAuthor == null) {
198
						logger.warn("Reference author is null");
199
					}
200
					author = Team.NewInstance();
201
					author.setTitleCache(refAuthor);
202
					authors.put(refAuthor,author); 
203
					if (logger.isTraceEnabled()) { 
204
						logger.trace("Stored author (" + refAuthor + ")");
205
					}
206
				//}
197 207

  
208
				} else {
209
					author = authors.get(refAuthor);
210
					if (logger.isDebugEnabled()) { 
211
						logger.debug("Not imported author with duplicated aut_id (" + refId + 
212
							") " + refAuthor);
213
					}
214
				}
215
				
216
				reference.setAuthorTeam(author);
217
				
198 218
				ImportHelper.setOriginalSource(reference, fauEuConfig.getSourceReference(), refId, namespace);
199 219
				ImportHelper.setOriginalSource(author, fauEuConfig.getSourceReference(), refId, namespace);
200 220

  
201 221
				// Store reference
202 222

  
203
				if (!references.contains(refId)) {
223

  
224
				if (!references.containsKey(refId)) {
225

  
204 226
					if (reference == null) {
205 227
						logger.warn("Reference is null");
206 228
					}
207
					references.add(reference);
229
					references.put(refId, reference);
208 230
					if (logger.isTraceEnabled()) { 
209 231
						logger.trace("Stored reference (" + refAuthor + ")"); 
210 232
					}
......
212 234
					if (logger.isDebugEnabled()) { 
213 235
						logger.debug("Duplicated reference (" + refId + ", " + refAuthor + ")");
214 236
					}
215
					continue;
237
					//continue;
216 238
				}
217

  
218
				// Store author
219

  
220
				boolean store = true;
221
				if (!authors.contains(refId)) {
222
					if (refAuthor == null) {
223
						logger.warn("Reference author is null");
239
				
240
				if (((i % limit) == 0 && i > 1 ) || i == count) { 
241
					
242
					Map <UUID, ReferenceBase> referenceMap =getReferenceService().save(references.values());
243
					logger.info("i = " + i + " - references saved"); 
244

  
245
					Iterator<Entry<UUID, ReferenceBase>> it = referenceMap.entrySet().iterator();
246
					while (it.hasNext()){
247
						ReferenceBase ref = it.next().getValue();
248
						int refID = Integer.valueOf(((OriginalSourceBase)ref.getSources().iterator().next()).getIdInSource());
249
						UUID uuid = ref.getUuid();
250
						referenceUuids.put(refID, uuid);
224 251
					}
225
					String storedAuthorTitleCache = null;
226
					for (TeamOrPersonBase<?> storedAuthor : authors) {
227
						storedAuthorTitleCache = storedAuthor.getTitleCache();
228
						if (storedAuthorTitleCache.equals(refAuthor)) {
229
							store = false;
230
							if (logger.isDebugEnabled()) {
231
								logger.debug("Duplicated author (" + refId + ", " + refAuthor + ")");
232
							}
233
							break;
252
					references= null;
253
					getAgentService().save((Collection)authors.values());
254
					
255
					authors = null;
256
				}
257
				
258
	        	
259
	        	
260
	        }
261
		}catch(SQLException e) {
262
			logger.error("SQLException:" +  e);
263
			success = false;
264
		}
265
	        
266
	        
267
	 //create the relationships between references and taxa       
268
	        
269
	        Taxon taxon = null;
270
	        i = 0;
271
	        try{
272
	        	ResultSet rsTaxRefs = source.getResultSet(countQueryTaxRefs);
273
	        	rsTaxRefs.next();
274
				count = rsTaxRefs.getInt(1);
275
				
276
				rsTaxRefs = source.getResultSet(selectQueryTaxRefs);
277
	        
278
				logger.info("Start taxon reference-relationships");
279
				FaunaEuropaeaReference fauEuReference;
280
				FaunaEuropaeaReferenceTaxon fauEuReferenceTaxon;
281
				while (rsTaxRefs.next()) {
282
					
283
					
284
					if ((i++ % limit) == 0) {
285
	
286
						txStatus = startTransaction();
287
						taxonUuids = new HashSet<UUID>(limit);
288
						referenceIDs = new HashSet<Integer>(limit);
289
						authors = new HashMap<String,TeamOrPersonBase>(limit);
290
						fauEuTaxonMap = new HashMap<UUID, FaunaEuropaeaReferenceTaxon>(limit);
291
	
292
						if(logger.isInfoEnabled()) {
293
							logger.info("i = " + i + " - Reference import transaction started"); 
234 294
						}
235 295
					}
236
					if (store == true) { 
237
						authors.add(author); 
296
					
297
	
298
					int taxonId = rsTaxRefs.getInt("trf_tax_id");
299
					int refId = rsTaxRefs.getInt("ref_id");
300
					String refAuthor = rsTaxRefs.getString("ref_author");
301
					String year = rsTaxRefs.getString("ref_year");
302
					String title = rsTaxRefs.getString("ref_title");
303
					
304
					if (year == null){
305
						try{		
306
							year = String.valueOf((Integer.parseInt(title)));
307
						}
308
						catch(Exception ex)   
309
						{
310
							logger.info("year is empty and " +title + " contains no integer");
311
					    }
312
					}
313
					String refSource = rsTaxRefs.getString("ref_source");
314
					String page = rsTaxRefs.getString("trf_page");
315
					UUID currentTaxonUuid = null;
316
					if (resultSetHasColumn(rsTaxRefs, "UUID")){
317
						currentTaxonUuid = UUID.fromString(rsTaxRefs.getString("UUID"));
318
					} else {
319
						logger.error("Taxon (" + taxonId + ") without UUID ignored");
320
						continue;
321
					}
322
	
323
					fauEuReference = new FaunaEuropaeaReference();
324
					fauEuReference.setTaxonUuid(currentTaxonUuid);
325
					fauEuReference.setReferenceId(refId);
326
					fauEuReference.setReferenceAuthor(refAuthor);
327
					fauEuReference.setReferenceYear(year);
328
					fauEuReference.setReferenceTitle(title);
329
					fauEuReference.setReferenceSource(refSource);
330
					fauEuReference.setPage(page);
331
	
332
					if (!taxonUuids.contains(currentTaxonUuid)) {
333
						taxonUuids.add(currentTaxonUuid);
334
						fauEuReferenceTaxon = 
335
							new FaunaEuropaeaReferenceTaxon(currentTaxonUuid);
336
						fauEuTaxonMap.put(currentTaxonUuid, fauEuReferenceTaxon);
337
					} else {
238 338
						if (logger.isTraceEnabled()) { 
239
							logger.trace("Stored author (" + refAuthor + ")");
339
							logger.trace("Taxon (" + currentTaxonUuid + ") already stored.");
340
							//continue; ein Taxon kann mehr als eine Referenz haben
240 341
						}
241 342
					}
242

  
243
				} else {
244
					if (logger.isDebugEnabled()) { 
245
						logger.debug("Not imported author with duplicated aut_id (" + refId + 
246
								") " + refAuthor);
343
	
344
					if (!referenceIDs.contains(refId)) {
345
	
346
						
347
						referenceIDs.add(refId);
348
						if (logger.isTraceEnabled()) { 
349
							logger.trace("Stored reference (" + refAuthor + ")"); 
350
						}
351
					} else {
352
						if (logger.isDebugEnabled()) { 
353
							logger.debug("Duplicated reference (" + refId + ", " + refAuthor + ")");
354
						}
355
						//continue;
247 356
					}
248
				}
249

  
250
				fauEuTaxonMap.get(currentTaxonUuid).addReference(fauEuReference);
251

  
252
				Taxon taxon = null;
253
				if (((i % limit) == 0 && i != 1 ) || i == count) { 
254

  
357
	
358
					
359
					fauEuTaxonMap.get(currentTaxonUuid).addReference(fauEuReference);
360
	
361
					
362
				
363
			
364
				if (((i % limit) == 0 && i > 1 ) || i == count) { 
365
	
255 366
					try {
256

  
367
	
257 368
						taxonList = getTaxonService().find(taxonUuids);
258

  
369
						//get UUIDs of used references
370
						Iterator itRefs = referenceIDs.iterator();
371
						Set<UUID> uuidSet = new HashSet<UUID>(referenceIDs.size());
372
						UUID uuid;
373
						while (itRefs.hasNext()){
374
							uuid = referenceUuids.get(itRefs.next());
375
							uuidSet.add(uuid);
376
						}
377
						referenceList = getReferenceService().find(uuidSet);
378
						references = new HashMap<Integer, ReferenceBase>(limit);
379
						for (ReferenceBase ref : referenceList){
380
							references.put(Integer.valueOf(((OriginalSourceBase)ref.getSources().iterator().next()).getIdInSource()), ref);
381
						}
259 382
						for (TaxonBase taxonBase : taxonList) {
260

  
383
	
261 384
							// Create descriptions
262

  
385
	
263 386
							if (taxonBase == null) { 
264 387
								if (logger.isDebugEnabled()) { 
265
									logger.debug("TaxonBase is null (" + currentTaxonUuid + ")");
388
									logger.debug("TaxonBase is null ");
266 389
								}
267 390
								continue; 
268 391
							}
......
273 396
								if (acceptedTaxa.size() > 0) {
274 397
									taxon = syn.getAcceptedTaxa().iterator().next();
275 398
								} else {
276
//									if (logger.isDebugEnabled()) { 
399
	//								if (logger.isDebugEnabled()) { 
277 400
										logger.warn("Synonym (" + taxonBase.getUuid() + ") does not have accepted taxa");
278
//									}
401
	//								}
279 402
								}
280 403
							} else {
281 404
								taxon = CdmBase.deproxy(taxonBase, Taxon.class);
282 405
							}
283

  
406
	
284 407
							if (taxon != null) {
285 408
								TaxonDescription taxonDescription = null;
286 409
								Set<TaxonDescription> descriptions = taxon.getDescriptions();
......
290 413
									taxonDescription = TaxonDescription.NewInstance();
291 414
									taxon.addDescription(taxonDescription);
292 415
								}
293

  
294

  
416
	
417
	
295 418
								UUID taxonUuid = taxonBase.getUuid();
296 419
								FaunaEuropaeaReferenceTaxon fauEuHelperTaxon = fauEuTaxonMap.get(taxonUuid);
297

  
420
								ReferenceBase citation;
421
								String microCitation;
422
								DescriptionElementSource originalSource;
423
								Synonym syn;
298 424
								for (FaunaEuropaeaReference storedReference : fauEuHelperTaxon.getReferences()) {
299

  
425
	
300 426
									TextData textData = TextData.NewInstance(Feature.CITATION());
301 427
									
302
									ReferenceBase citation = storedReference.getCdmReference();
303
									String microCitation = storedReference.getPage();
304
									DescriptionElementSource originalSource = DescriptionElementSource.NewInstance(null, null, citation, microCitation, null, null);
428
									citation = references.get(storedReference.getReferenceId());
429
									microCitation = storedReference.getPage();
430
									originalSource = DescriptionElementSource.NewInstance(null, null, citation, microCitation, null, null);
305 431
									if (isSynonym){
306
										Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
432
										syn = CdmBase.deproxy(taxonBase, Synonym.class);
307 433
										originalSource.setNameUsedInSource(syn.getName());
308 434
									}
309 435
									textData.addSource(originalSource);
......
314 440
						if(logger.isInfoEnabled()) { 
315 441
							logger.info("i = " + i + " - Transaction committed"); 
316 442
						}
317

  
318
						// save taxa, references, and authors
443
	
444
						// save taxa
319 445
						getTaxonService().save(taxonList);
320
						getReferenceService().save(references);
321
						getAgentService().save((Collection)authors);
322

  
446
						
447
	
323 448
						taxonUuids = null;
324 449
						references = null;
325
						authors = null;
326 450
						taxonList = null;
327 451
						fauEuTaxonMap = null;
452
						referenceIDs = null;
453
						referenceList = null;
454
						uuidSet = null;
328 455
						commitTransaction(txStatus);
329

  
456
	
330 457
					} catch (Exception e) {
331
						logger.warn("An exception occurred when creating reference with id " + refId + 
332
						". Reference could not be saved.");
458
						logger.warn("An exception occurred when creating reference, reference could not be saved.");
333 459
					}
334 460
				}
335
			}
461
				}
462
			rsTaxRefs.close();
336 463
		} catch (SQLException e) {
337 464
			logger.error("SQLException:" +  e);
338 465
			success = false;
339 466
		}
467
		
340 468
		if(logger.isInfoEnabled()) { logger.info("End making references ..."); }
341

  
469
		
342 470
		return success;
343 471
	}
344 472

  
473
	
474
	
475

  
476
	
345 477

  
346 478
	/* (non-Javadoc)
347 479
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)

Also available in: Unified diff