Project

General

Profile

« Previous | Next » 

Revision a399b702

Added by U-BGBM\k.luther over 8 years ago

changes for fauna europaea import

View differences:

cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaVernacularNamesImport.java
14 14
import java.util.Map.Entry;
15 15

  
16 16
import org.apache.log4j.Logger;
17
import org.springframework.stereotype.Component;
17 18
import org.springframework.transaction.TransactionStatus;
18 19

  
19 20
import eu.etaxonomy.cdm.io.common.Source;
21
import eu.etaxonomy.cdm.jaxb.TDWGNamespacePrefixMapper;
20 22
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
21 23
import eu.etaxonomy.cdm.model.common.CdmBase;
22 24
import eu.etaxonomy.cdm.model.common.Language;
23 25
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
26
import eu.etaxonomy.cdm.model.common.TermType;
24 27
import eu.etaxonomy.cdm.model.common.TermVocabulary;
25 28
import eu.etaxonomy.cdm.model.description.CommonTaxonName;
26 29
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
......
30 33
import eu.etaxonomy.cdm.model.location.NamedArea;
31 34
import eu.etaxonomy.cdm.model.reference.Reference;
32 35
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
36
import eu.etaxonomy.cdm.model.taxon.Synonym;
33 37
import eu.etaxonomy.cdm.model.taxon.Taxon;
34 38
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
35 39

  
40
@Component
36 41
public class FaunaEuropaeaVernacularNamesImport extends FaunaEuropaeaImportBase {
37 42
	private static final Logger logger = Logger.getLogger(FaunaEuropaeaVernacularNamesImport.class);
38 43

  
......
42 47
	protected void doInvoke(FaunaEuropaeaImportState state) {
43 48
		int limit = state.getConfig().getLimitSave();
44 49
		
45
		/* Taxon store for retrieving taxa from and saving taxa to CDM */
46
		List<TaxonBase> taxonList = null;
47
		/* UUID store as input for retrieving taxa from CDM */
48
		Set<UUID> taxonUuids = null;
49
		/* Store to hold helper objects */
50
		Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuVernacularNamesMap = null;
51

  
52

  
53
		TransactionStatus txStatus = null;
54
		
55
		createSourceMap(state);
56
		
57
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
58
		Source source = fauEuConfig.getSource();
59

  
60
        int i = 0;
61

  
62
		String selectCount =
63
			" SELECT count(*) ";
64

  
65
		String selectColumns =
66
			" SELECT vernacularName.*, Taxon.UUID ";
67

  
68
		String fromClause =
69
			" FROM vernacularName INNER JOIN " +
70
            " Taxon ON vernacularname.taxon_UUID = Taxon.UUID ";
71
		String orderBy = " ORDER BY vernacularName.taxon_UUID";
72

  
73
		String countQuery =
74
			selectCount + fromClause;
75

  
76
		String selectQuery =
77
			selectColumns + fromClause + orderBy;
78

  
79

  
80

  
81
		if(logger.isInfoEnabled()) { logger.info("Start making distributions..."); }
82

  
83
		try {
84
			ResultSet rs = source.getResultSet(countQuery);
85
			rs.next();
86
			int count = rs.getInt(1);
87

  
88
			rs = source.getResultSet(selectQuery);
89

  
90
			if (logger.isInfoEnabled()) {
91
				logger.info("Number of rows: " + count);
92
				logger.info("Count Query: " + countQuery);
93
				logger.info("Select Query: " + selectQuery);
94
			}
95

  
96
			//int taxonId;
97

  
98
			while (rs.next()) {
99
				if ((i++ % limit) == 0) {
100

  
101
					txStatus = startTransaction();
102
					taxonUuids = new HashSet<UUID>(limit);
103
					fauEuVernacularNamesMap = new HashMap<UUID, FaunaEuropaeaVernacularNamesTaxon>(limit);
104

  
105
					if(logger.isInfoEnabled()) {
106
						logger.info("i = " + i + " - Distribution import transaction started");
107
					}
50
		if (state.getConfig().isDoVernacularNames()){
51
			/* Taxon store for retrieving taxa from and saving taxa to CDM */
52
			List<TaxonBase> taxonList = null;
53
			/* UUID store as input for retrieving taxa from CDM */
54
			Set<UUID> taxonUuids = null;
55
			/* Store to hold helper objects */
56
			Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuVernacularNamesMap = null;
57
	
58
	
59
			TransactionStatus txStatus = null;
60
			
61
			createSourceMap(state);
62
			
63
			FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
64
			Source source = fauEuConfig.getSource();
65
	
66
	        int i = 0;
67
	
68
			String selectCount =
69
				" SELECT count(*) ";
70
	
71
			String selectColumns =
72
				" SELECT vernacularNames.*, Taxon.UUID ";
73
	
74
			String fromClause =
75
				" FROM vernacularNames INNER JOIN " +
76
	            " Taxon ON vernacularnames.UUID = Taxon.UUID ";
77
			String orderBy = " ORDER BY vernacularNames.UUID";
78
	
79
			String countQuery =
80
				selectCount + fromClause;
81
	
82
			String selectQuery =
83
				selectColumns + fromClause + orderBy;
84
	
85
	
86
	
87
			if(logger.isInfoEnabled()) { logger.info("Start making vernacular names..."); }
88
	
89
			try {
90
				ResultSet rs = source.getResultSet(countQuery);
91
				rs.next();
92
				int count = rs.getInt(1);
93
	
94
				rs = source.getResultSet(selectQuery);
95
	
96
				if (logger.isInfoEnabled()) {
97
					logger.info("Number of rows: " + count);
98
					logger.info("Count Query: " + countQuery);
99
					logger.info("Select Query: " + selectQuery);
108 100
				}
109

  
110
				//taxonId = rs.getInt("dis_tax_id");
111
				String vernameGUID = rs.getString("GUID");
112
				String vernacularNameString = rs.getString("verName");
113
				int taxonFk = rs.getInt("taxonFK");
114
				int languageFK= rs.getInt("languageFK");
115
				String languageCache = rs.getString("languageCache");
116
				String taxonUUID= rs.getString("taxonUUID");
117
				String verSource = rs.getString("VerSource");
118
				UUID currentTaxonUuid = null;
119
				
120

  
121
				FaunaEuropaeaVernacularName fauEuVernacularName = new FaunaEuropaeaVernacularName();
122
				fauEuVernacularName.setGuid(vernameGUID);
123
				fauEuVernacularName.setLanguageCache(languageCache);
124
				fauEuVernacularName.setLanguageFk(languageFK);
125
				fauEuVernacularName.setVernacularName(vernacularNameString);
126
				fauEuVernacularName.setSource(verSource);
127

  
128
				if (!taxonUuids.contains(currentTaxonUuid)) {
129
					taxonUuids.add(currentTaxonUuid);
130
					FaunaEuropaeaVernacularNamesTaxon fauEuVernacularNameTaxon =
131
						new FaunaEuropaeaVernacularNamesTaxon(currentTaxonUuid);
132
					fauEuVernacularNamesMap.put(currentTaxonUuid, fauEuVernacularNameTaxon);
133
					fauEuVernacularNameTaxon = null;
134
				} else {
135
					if (logger.isTraceEnabled()) {
136
						logger.trace("Taxon (" + currentTaxonUuid + ") already stored.");
137
						continue;
101
	
102
				//int taxonId;
103
	
104
				while (rs.next()) {
105
					if ((i++ % limit) == 0) {
106
	
107
						txStatus = startTransaction();
108
						taxonUuids = new HashSet<UUID>(limit);
109
						fauEuVernacularNamesMap = new HashMap<UUID, FaunaEuropaeaVernacularNamesTaxon>(limit);
110
	
111
						if(logger.isInfoEnabled()) {
112
							logger.info("i = " + i + " - vernacular names import transaction started");
113
						}
114
					}
115
	
116
					//taxonId = rs.getInt("dis_tax_id");
117
					String vernameGUID = rs.getString("GUID");
118
					String vernacularNameString = rs.getString("verName");
119
					//int taxonFk = rs.getInt("taxonFK");
120
					int languageFK= rs.getInt("languageFK");
121
					String languageCache = rs.getString("languageCache");
122
					String taxonUUID= rs.getString("UUID");
123
					String verSource = rs.getString("VerSource");
124
					UUID currentTaxonUuid = null;
125
					
126
	
127
					FaunaEuropaeaVernacularName fauEuVernacularName = new FaunaEuropaeaVernacularName();
128
					fauEuVernacularName.setGuid(vernameGUID);
129
					fauEuVernacularName.setLanguageCache(languageCache);
130
					fauEuVernacularName.setLanguageFk(languageFK);
131
					fauEuVernacularName.setVernacularName(vernacularNameString);
132
					fauEuVernacularName.setSource(verSource);
133
					fauEuVernacularName.setTaxonUuid(UUID.fromString(taxonUUID));
134
					currentTaxonUuid =UUID.fromString(taxonUUID);
135
					if (!taxonUuids.contains(currentTaxonUuid)) {
136
						taxonUuids.add(currentTaxonUuid);
137
						FaunaEuropaeaVernacularNamesTaxon fauEuVernacularNameTaxon =
138
							new FaunaEuropaeaVernacularNamesTaxon(currentTaxonUuid);
139
						fauEuVernacularNamesMap.put(currentTaxonUuid, fauEuVernacularNameTaxon);
140
						fauEuVernacularNameTaxon = null;
141
					} else {
142
						if (logger.isTraceEnabled()) {
143
							logger.trace("Taxon (" + currentTaxonUuid + ") already stored.");
144
							continue;
145
						}
146
					}
147
	
148
					fauEuVernacularNamesMap.get(currentTaxonUuid).addVernacularName(fauEuVernacularName);
149
	
150
					if (((i % limit) == 0 && i != 1 ) || i == count ) {
151
	
152
						try {
153
							commitTaxaAndVernacularNames(state,  taxonUuids, fauEuVernacularNamesMap, txStatus);
154
							taxonUuids = null;
155
							taxonList = null;
156
							fauEuVernacularNamesMap = null;
157
	
158
						} catch (Exception e) {
159
							logger.error("Commit of taxa and distributions failed" + e.getMessage());
160
							e.printStackTrace();
161
						}
162
	
163
						if(logger.isInfoEnabled()) { logger.info("i = " + i + " - Transaction committed");}
138 164
					}
165
	
166
	
139 167
				}
140

  
141
				fauEuVernacularNamesMap.get(currentTaxonUuid).addVernacularName(fauEuVernacularName);
142

  
143
				if (((i % limit) == 0 && i != 1 ) || i == count ) {
144

  
168
				if (taxonUuids != null){
145 169
					try {
146
						commitTaxaAndVernacularNames(state,  taxonUuids, fauEuVernacularNamesMap, txStatus);
170
						commitTaxaAndVernacularNames(state, taxonUuids, fauEuVernacularNamesMap, txStatus);
147 171
						taxonUuids = null;
148 172
						taxonList = null;
149 173
						fauEuVernacularNamesMap = null;
150

  
151 174
					} catch (Exception e) {
152
						logger.error("Commit of taxa and distributions failed" + e.getMessage());
175
						logger.error("Commit of taxa and vernacular names failed");
176
						logger.error(e.getMessage());
153 177
						e.printStackTrace();
154 178
					}
155

  
156
					if(logger.isInfoEnabled()) { logger.info("i = " + i + " - Transaction committed");}
157
				}
158

  
159

  
160
			}
161
			if (taxonUuids != null){
162
				try {
163
					commitTaxaAndVernacularNames(state, taxonUuids, fauEuVernacularNamesMap, txStatus);
164
					taxonUuids = null;
165
					taxonList = null;
166
					fauEuVernacularNamesMap = null;
167
				} catch (Exception e) {
168
					logger.error("Commit of taxa and distributions failed");
169
					logger.error(e.getMessage());
170
					e.printStackTrace();
171 179
				}
180
				rs = null;
181
			} catch (SQLException e) {
182
				logger.error("SQLException:" +  e);
183
				state.setUnsuccessfull();
172 184
			}
173
			rs = null;
174
		} catch (SQLException e) {
175
			logger.error("SQLException:" +  e);
176
			state.setUnsuccessfull();
185
	
186
			if(logger.isInfoEnabled()) { logger.info("End making vernacular names..."); }
187
	
188
			return;
177 189
		}
178 190

  
179
		if(logger.isInfoEnabled()) { logger.info("End making distributions..."); }
180

  
181
		return;
182

  
183 191
	}
184 192

  
185 193
	@Override
......
198 206
			Set<UUID> taxonUuids,
199 207
			Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuTaxonMap,
200 208
			TransactionStatus txStatus) throws Exception {
201
		 List<TaxonBase> taxonList = prepareTaxaAndVernacularNames(getTaxonService().find(taxonUuids), fauEuTaxonMap, state);
209
		List<TaxonBase> taxonList = prepareTaxaAndVernacularNames(getTaxonService().find(taxonUuids), fauEuTaxonMap, state);
202 210

  
203
		getTaxonService().save(taxonList);
211
		getTaxonService().saveOrUpdate(taxonList);
204 212
		taxonList = null;
205 213
		taxonUuids = null;
206 214
		fauEuTaxonMap = null;
......
224 232
				if (taxonBase instanceof Taxon) {
225 233
					taxon = CdmBase.deproxy(taxonBase, Taxon.class);
226 234
				} else {
235
					Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
236
					if (syn.getAcceptedTaxa().size()>0){
237
						taxon = syn.getAcceptedTaxa().iterator().next();
238
					}
227 239
					logger.warn("TaxonBase (" + taxonBase.getId() + " is not of type Taxon but: "
228
							+ taxonBase.getClass().getSimpleName());
240
							+ taxonBase.getClass().getSimpleName() + " using accepted Taxon for vernacular name");
229 241
					continue;
230 242
				}
231 243

  
232 244

  
233 245
				Set<TaxonDescription> descriptionSet = taxon.getDescriptions();
234
				if (descriptionSet.size() > 0) {
235
					taxonDescription = descriptionSet.iterator().next();
236
				} else {
237
					taxonDescription = TaxonDescription.NewInstance();
238
					taxon.addDescription(taxonDescription);
239
				}
246
				taxonDescription = TaxonDescription.NewInstance();
247
				taxon.addDescription(taxonDescription);
248
				
240 249

  
241 250
				taxonUuid = taxonBase.getUuid();
242 251
				fauEuHelperTaxon= fauEuTaxonMap.get(taxonUuid);
......
244 253
				for (FaunaEuropaeaVernacularName fauEuHelperVernacularName: fauEuHelperTaxon.getVernacularNames()) {
245 254
					
246 255
					verName = null;
256
					
257
					/*
258
					 * UUID uuid = state.getTransformer().getLanguageUuid(languageKey);
259
					result = (Language)getTermService().find(uuid);
260
				}
261
				if (result == null){
262
					result = state.getTransformer().getLanguageByKey(languageKey);
263
					if (result == null){
264
						UUID uuidLanguage;
265
						uuidLanguage = state.getTransformer().getLanguageUuid(languageKey);
266
						if (uuidLanguage == null){
267
							logger.warn("Language not defined: " + languageKey)  ;
268
						}
269
						result = getLanguage(state, uuidLanguage, languageKey, languageKey, null);
270
						if (result == null){
271
							logger.warn("Language not defined: " + languageKey)  ;
272
						}
273
					}else if (result.getId() == 0){
274
//						UUID uuidLanguageVoc = UUID.fromString("45ac7043-7f5e-4f37-92f2-3874aaaef2de"); 
275
						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
276
						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, result);
277
//						TermVocabulary<Language> voc = getVocabularyService().find(uuidLanguageVoc);
278
						voc.addTerm(result);
279
						getTermService().saveOrUpdate(result);
280
						state.putLanguage(result);
281
					}
282
					 * 
283
					 */
247 284
					Language lang = FaunaEuropaeaTransformer.langFK2Language(fauEuHelperVernacularName.getLanguageFk());
285
					if (lang == null && fauEuHelperVernacularName.getLanguageFk() == 83){
286
						lang = Language.ENGLISH();
287
						fauEuHelperVernacularName.setArea("USA");
288
					} 
289
					
290
					if (lang == null){
291
						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
292
						logger.info("languageFk = " + fauEuHelperVernacularName.getLanguageFk());
293
						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, lang);
294
						lang = Language.NewInstance("Dummy", "", "");
295
						voc.addTerm(lang);
296
						lang =(Language)getTermService().save(lang);
297
						FaunaEuropaeaTransformer.languageFK2Language.put(fauEuHelperVernacularName.getLanguageFk(), lang);
298
					}
299
//					if (lang.getId() == 0){
300
//						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
301
//						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, lang);
302
//						voc.addTerm(lang);
303
//						lang =(Language)getTermService().save(lang);
304
//						//FaunaEuropaeaTransformer.languageFK2Language.put(fauEuHelperVernacularName.getLanguageFk(), lang);
305
//					}
306
					
248 307
					verName = CommonTaxonName.NewInstance(fauEuHelperVernacularName.getVernacularName(), lang);
308
					if (fauEuHelperVernacularName.getArea()!= null && fauEuHelperVernacularName.getArea().equals("USA")){
309
						verName.setArea(NamedArea.NORTH_AMERICA());
310
					}
249 311
					verName.setCreated(null);
250 312
					addOriginalSource(verName, null, null, sourceMap.get(fauEuHelperVernacularName.getSource()));
251 313
					taxonDescription.addElement(verName);
......
260 322
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
261 323
		Source source = fauEuConfig.getSource();
262 324
		TransactionStatus txStatus = null;
263
		String sourceSelect = "SELECT DISTINCT PESI_FaEu_vernaculars_export.VerSource FROM PESI_FaEu_vernaculars_export group by PESI_FaEu_vernaculars_export.VerSource";
325
		String sourceSelect = "SELECT DISTINCT vernacularNames.VerSource FROM vernacularNames group by vernacularNames.VerSource";
264 326
		try {
265 327
			txStatus =startTransaction();
266 328
			ResultSet rs = source.getResultSet(sourceSelect);
......
291 353
			String refID = ref.getTitle();
292 354
			sourceMap.put(refID, ref);
293 355
		}
356
		
294 357
		commitTransaction(txStatus);
295 358
	}
296 359
}

Also available in: Unified diff