Project

General

Profile

« Previous | Next » 

Revision 79997ad8

Added by U-BGBM\k.luther over 8 years ago

Fauna Europaea import changes

View differences:

cdm-pesi/src/main/java/eu/etaxonomy/cdm/app/common/CdmDestinations.java
34 34
	public static ICdmDataSource cdm_test_local_faunaEu_mysql(){
35 35
		DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
36 36
		String cdmServer = "127.0.0.1";
37
		String cdmDB = "cdm_faunaeu_2015_09";
37
		String cdmDB = "cdm_faunaeu_2015_11";
38 38
		String cdmUserName = "root";
39 39
		return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null);
40 40
	}
cdm-pesi/src/main/java/eu/etaxonomy/cdm/app/pesi/FaunaEuropaeaActivator.java
47 47

  
48 48
//	static final CHECK check = CHECK.CHECK_AND_IMPORT;
49 49
	static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
50
	static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.CREATE;
50
//	static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.CREATE;
51 51
//	static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.UPDATE;
52
//	static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.VALIDATE;
52
	static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.VALIDATE;
53 53

  
54 54

  
55 55
	
......
58 58
// ****************** ALL *****************************************
59 59

  
60 60
	// Fauna Europaea to CDM import
61
	static final boolean doAuthors = true;
62
	static final boolean doTaxa = true;
63
	static final boolean doBasionyms = true;
64
	static final boolean doTaxonomicallyIncluded = true;
65
	static final boolean doMisappliedNames = true;
66
	static final boolean doHeterotypicSynonyms = true;
61
	static final boolean doAuthors = false;
62
	static final boolean doTaxa = false;
63
	static final boolean doBasionyms = false;
64
	static final boolean doTaxonomicallyIncluded = false;
65
	static final boolean doMisappliedNames = false;
66
	static final boolean doHeterotypicSynonyms = false;
67 67
	static final DO_REFERENCES doReferences =  DO_REFERENCES.ALL;
68
	static final boolean doDistributions = true;
69
	static final boolean makeFeatureTree = true;
68
	static final boolean doDistributions = false;
69
	static final boolean makeFeatureTree = false;
70 70
	static final boolean doVernacularNames = true;
71
	static final boolean doAssociatedSpecialists = true;
72
	static final boolean addCommonNameFeature = true;
71
	static final boolean doAssociatedSpecialists = false;
72
	static final boolean addCommonNameFeature = false;
73 73
    // CDM to CDM import
74
	static final boolean doHeterotypicSynonymsForBasionyms = true;
74
	static final boolean doHeterotypicSynonymsForBasionyms = false;
75 75
	
76 76
// ************************ NONE **************************************** //
77 77

  
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
67 67
				FaunaEuropaeaUsersImport.class,
68 68
				FaunaEuropaeaTaxonNameImport.class,
69 69
				FaunaEuropaeaRelTaxonIncludeImport.class,
70
				FaunaEuropaeaRefImport.class,
71

  
72 70
				FaunaEuropaeaDistributionImport.class,
73 71
				FaunaEuropaeaHeterotypicSynonymImport.class,
72
				FaunaEuropaeaRefImport.class,
74 73
				FaunaEuropaeaAdditionalTaxonDataImport.class,
75 74
				FaunaEuropaeaVernacularNamesImport.class
76 75
		};
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaRefImport.java
209 209
//				reference.setTitleCache(title);
210 210
				reference.setTitle(title);
211 211
				reference.setDatePublished(ImportHelper.getDatePublished(year));
212
			
212
				reference.setTitleCache(title + " " +refSource , true);
213 213

  
214 214
				if (!authors.containsKey(refAuthor)) {
215 215
					if (refAuthor == null) {
......
281 281
	private void commitReferences(Map<Integer, Reference<?>> references,
282 282
			Map<String, TeamOrPersonBase<?>> authors,
283 283
			Map<Integer, UUID> referenceUuids, int i, TransactionStatus txStatus) {
284
		
284 285
		Map <UUID, Reference> referenceMap =getReferenceService().save((Collection)references.values());
285 286
		logger.info("i = " + i + " - references saved");
286 287

  
......
446 447
			uuid = referenceUuids.get(itRefs.next());
447 448
			uuidSet.add(uuid);
448 449
		}
450
		
449 451
		referenceList = getReferenceService().find(uuidSet);
452
		
450 453
		references = new HashMap<Integer, Reference>(limit);
451 454
		for (Reference<?> ref : referenceList){
452 455
			references.put(Integer.valueOf(((OriginalSourceBase)ref.getSources().iterator().next()).getIdInSource()), ref);
......
499 502
					TextData textData = TextData.NewInstance(Feature.CITATION());
500 503

  
501 504
					citation = references.get(storedReference.getReferenceId());
502
					microCitation = storedReference.getReferenceSource();
505
					microCitation = null;
503 506
					originalSource = DescriptionElementSource.NewInstance(OriginalSourceType.PrimaryTaxonomicSource, null, null, citation, microCitation, null, null);
504 507
					if (isSynonym){
505 508
						syn = CdmBase.deproxy(taxonBase, Synonym.class);
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaTransformer.java
245 245
            } else if (areaCode.equals("EE")) {
246 246
                tdwgArea = TdwgAreaProvider.getAreaByTdwgAbbreviation("BLT-ES");
247 247
            } else if (areaCode.equals("ES-BAL")) {
248
                tdwgArea = TdwgAreaProvider.getAreaByTdwgAbbreviation("BLT-ES");
248
                tdwgArea = TdwgAreaProvider.getAreaByTdwgAbbreviation("BAL-OO");
249 249
            } else if (areaCode.equals("ES-CNY")) {
250 250
                tdwgArea = TdwgAreaProvider.getAreaByTdwgAbbreviation("CNY-OO");
251 251
            } else if (areaCode.equals("ES-SPA")) {
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaUsersImport.java
86 86
		ProfilerController.memorySnapshot();
87 87
		*/
88 88
		
89
		
89
		if (!state.getConfig().isDoAuthors()){
90
			return;
91
		}
90 92
		TransactionStatus txStatus = null;
91 93
		Map<String, Person> persons = null;
92 94
		Map<String, User> users= null;
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaVernacularName.java
1
package eu.etaxonomy.cdm.io.pesi.faunaEuropaea;
2

  
3
import java.util.UUID;
4

  
5
public class FaunaEuropaeaVernacularName {
6
	
7
	String vernacularName;
8
	int languageFk;
9
	String languageCache;
10
	UUID taxonUuid;
11
	String guid;
12
	String source;
13
	String area = "";
14
	
15
	public String getArea() {
16
		return area;
17
	}
18

  
19
	public void setArea(String area) {
20
		this.area = area;
21
	}
22

  
23
	public String getVernacularName() {
24
		return vernacularName;
25
	}
26

  
27
	public void setVernacularName(String vernacularName) {
28
		this.vernacularName = vernacularName;
29
	}
30

  
31
	public int getLanguageFk() {
32
		return languageFk;
33
	}
34

  
35
	public void setLanguageFk(int languageFk) {
36
		this.languageFk = languageFk;
37
	}
38

  
39
	public String getLanguageCache() {
40
		return languageCache;
41
	}
42

  
43
	public void setLanguageCache(String languageCache) {
44
		this.languageCache = languageCache;
45
	}
46

  
47
	public UUID getTaxonUuid() {
48
		return taxonUuid;
49
	}
50

  
51
	public void setTaxonUuid(UUID taxonUuid) {
52
		this.taxonUuid = taxonUuid;
53
	}
54

  
55
	public String getGuid() {
56
		return guid;
57
	}
58

  
59
	public void setGuid(String guid) {
60
		this.guid = guid;
61
	}
62

  
63
	public String getSource() {
64
		return source;
65
	}
66

  
67
	public void setSource(String source) {
68
		this.source = source;
69
	}
70
	
71
	public FaunaEuropaeaVernacularName(String vernacularName, int LanguageFK, String languageCache, UUID taxonUuid, String guid, String source){
72
		this.guid=guid;
73
		this.languageCache = languageCache;
74
		this.languageFk = languageFk;
75
		this.taxonUuid = taxonUuid;
76
		this.vernacularName = vernacularName;
77
		this.source = source;
78
	
79
	}
80

  
81
	public FaunaEuropaeaVernacularName() {
82
		
83
	}
84
	
85
}
1
package eu.etaxonomy.cdm.io.pesi.faunaEuropaea;
2

  
3
import java.util.UUID;
4

  
5
public class FaunaEuropaeaVernacularName {
6
	
7
	String vernacularName;
8
	int languageFk;
9
	String languageCache;
10
	UUID taxonUuid;
11
	String guid;
12
	String source;
13
	String area = "";
14
	
15
	
16
	public FaunaEuropaeaVernacularName(String vernacularName, int languageFK, String languageCache, UUID taxonUuid, String guid, String source){
17
		this.guid=guid;
18
		this.languageCache = languageCache;
19
		this.languageFk = languageFk;
20
		this.taxonUuid = taxonUuid;
21
		this.vernacularName = vernacularName;
22
		this.source = source;
23
	
24
	}
25

  
26
	public FaunaEuropaeaVernacularName() {
27
		
28
	}
29
	
30
	
31
	public String getArea() {
32
		return area;
33
	}
34

  
35
	public void setArea(String area) {
36
		this.area = area;
37
	}
38

  
39
	public String getVernacularName() {
40
		return vernacularName;
41
	}
42

  
43
	public void setVernacularName(String vernacularName) {
44
		this.vernacularName = vernacularName;
45
	}
46

  
47
	public int getLanguageFk() {
48
		return languageFk;
49
	}
50

  
51
	public void setLanguageFk(int languageFk) {
52
		this.languageFk = languageFk;
53
	}
54

  
55
	public String getLanguageCache() {
56
		return languageCache;
57
	}
58

  
59
	public void setLanguageCache(String languageCache) {
60
		this.languageCache = languageCache;
61
	}
62

  
63
	public UUID getTaxonUuid() {
64
		return taxonUuid;
65
	}
66

  
67
	public void setTaxonUuid(UUID taxonUuid) {
68
		this.taxonUuid = taxonUuid;
69
	}
70

  
71
	public String getGuid() {
72
		return guid;
73
	}
74

  
75
	public void setGuid(String guid) {
76
		this.guid = guid;
77
	}
78

  
79
	public String getSource() {
80
		return source;
81
	}
82

  
83
	public void setSource(String source) {
84
		this.source = source;
85
	}
86
	
87
	
88
}
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaVernacularNamesImport.java
1
package eu.etaxonomy.cdm.io.pesi.faunaEuropaea;
2

  
3
import java.sql.ResultSet;
4
import java.sql.SQLException;
5
import java.util.ArrayList;
6
import java.util.Collection;
7
import java.util.HashMap;
8
import java.util.HashSet;
9
import java.util.Iterator;
10
import java.util.List;
11
import java.util.Map;
12
import java.util.Set;
13
import java.util.UUID;
14
import java.util.Map.Entry;
15

  
16
import org.apache.log4j.Logger;
17
import org.springframework.stereotype.Component;
18
import org.springframework.transaction.TransactionStatus;
19

  
20
import eu.etaxonomy.cdm.io.common.Source;
21
import eu.etaxonomy.cdm.jaxb.TDWGNamespacePrefixMapper;
22
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
23
import eu.etaxonomy.cdm.model.common.CdmBase;
24
import eu.etaxonomy.cdm.model.common.Language;
25
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
26
import eu.etaxonomy.cdm.model.common.TermType;
27
import eu.etaxonomy.cdm.model.common.TermVocabulary;
28
import eu.etaxonomy.cdm.model.description.CommonTaxonName;
29
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
30
import eu.etaxonomy.cdm.model.description.Distribution;
31
import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
32
import eu.etaxonomy.cdm.model.description.TaxonDescription;
33
import eu.etaxonomy.cdm.model.location.NamedArea;
34
import eu.etaxonomy.cdm.model.reference.Reference;
35
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
36
import eu.etaxonomy.cdm.model.taxon.Synonym;
37
import eu.etaxonomy.cdm.model.taxon.Taxon;
38
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
39

  
40
@Component
41
public class FaunaEuropaeaVernacularNamesImport extends FaunaEuropaeaImportBase {
42
	private static final Logger logger = Logger.getLogger(FaunaEuropaeaVernacularNamesImport.class);
43

  
44
	private HashMap<String, Reference> sourceMap = new HashMap<String, Reference>();
45
	
46
	@Override
47
	protected void doInvoke(FaunaEuropaeaImportState state) {
48
		int limit = state.getConfig().getLimitSave();
49
		
50
		if (state.getConfig().isDoVernacularNames()){
51
			/* Taxon store for retrieving taxa from and saving taxa to CDM */
52
			List<TaxonBase> taxonList = null;
53
			/* UUID store as input for retrieving taxa from CDM */
54
			Set<UUID> taxonUuids = null;
55
			/* Store to hold helper objects */
56
			Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuVernacularNamesMap = null;
57
	
58
	
59
			TransactionStatus txStatus = null;
60
			
61
			createSourceMap(state);
62
			
63
			FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
64
			Source source = fauEuConfig.getSource();
65
	
66
	        int i = 0;
67
	
68
			String selectCount =
69
				" SELECT count(*) ";
70
	
71
			String selectColumns =
72
				" SELECT vernacularNames.*, Taxon.UUID ";
73
	
74
			String fromClause =
75
				" FROM vernacularNames INNER JOIN " +
76
	            " Taxon ON vernacularnames.UUID = Taxon.UUID ";
77
			String orderBy = " ORDER BY vernacularNames.UUID";
78
	
79
			String countQuery =
80
				selectCount + fromClause;
81
	
82
			String selectQuery =
83
				selectColumns + fromClause + orderBy;
84
	
85
	
86
	
87
			if(logger.isInfoEnabled()) { logger.info("Start making vernacular names..."); }
88
	
89
			try {
90
				ResultSet rs = source.getResultSet(countQuery);
91
				rs.next();
92
				int count = rs.getInt(1);
93
	
94
				rs = source.getResultSet(selectQuery);
95
	
96
				if (logger.isInfoEnabled()) {
97
					logger.info("Number of rows: " + count);
98
					logger.info("Count Query: " + countQuery);
99
					logger.info("Select Query: " + selectQuery);
100
				}
101
	
102
				//int taxonId;
103
	
104
				while (rs.next()) {
105
					if ((i++ % limit) == 0) {
106
	
107
						txStatus = startTransaction();
108
						taxonUuids = new HashSet<UUID>(limit);
109
						fauEuVernacularNamesMap = new HashMap<UUID, FaunaEuropaeaVernacularNamesTaxon>(limit);
110
	
111
						if(logger.isInfoEnabled()) {
112
							logger.info("i = " + i + " - vernacular names import transaction started");
113
						}
114
					}
115
	
116
					//taxonId = rs.getInt("dis_tax_id");
117
					String vernameGUID = rs.getString("GUID");
118
					String vernacularNameString = rs.getString("verName");
119
					//int taxonFk = rs.getInt("taxonFK");
120
					int languageFK= rs.getInt("languageFK");
121
					String languageCache = rs.getString("languageCache");
122
					String taxonUUID= rs.getString("UUID");
123
					String verSource = rs.getString("VerSource");
124
					UUID currentTaxonUuid = null;
125
					
126
	
127
					FaunaEuropaeaVernacularName fauEuVernacularName = new FaunaEuropaeaVernacularName();
128
					fauEuVernacularName.setGuid(vernameGUID);
129
					fauEuVernacularName.setLanguageCache(languageCache);
130
					fauEuVernacularName.setLanguageFk(languageFK);
131
					fauEuVernacularName.setVernacularName(vernacularNameString);
132
					fauEuVernacularName.setSource(verSource);
133
					fauEuVernacularName.setTaxonUuid(UUID.fromString(taxonUUID));
134
					currentTaxonUuid =UUID.fromString(taxonUUID);
135
					if (!taxonUuids.contains(currentTaxonUuid)) {
136
						taxonUuids.add(currentTaxonUuid);
137
						FaunaEuropaeaVernacularNamesTaxon fauEuVernacularNameTaxon =
138
							new FaunaEuropaeaVernacularNamesTaxon(currentTaxonUuid);
139
						fauEuVernacularNamesMap.put(currentTaxonUuid, fauEuVernacularNameTaxon);
140
						fauEuVernacularNameTaxon = null;
141
					} else {
142
						if (logger.isTraceEnabled()) {
143
							logger.trace("Taxon (" + currentTaxonUuid + ") already stored.");
144
							continue;
145
						}
146
					}
147
	
148
					fauEuVernacularNamesMap.get(currentTaxonUuid).addVernacularName(fauEuVernacularName);
149
	
150
					if (((i % limit) == 0 && i != 1 ) || i == count ) {
151
	
152
						try {
153
							commitTaxaAndVernacularNames(state,  taxonUuids, fauEuVernacularNamesMap, txStatus);
154
							taxonUuids = null;
155
							taxonList = null;
156
							fauEuVernacularNamesMap = null;
157
	
158
						} catch (Exception e) {
159
							logger.error("Commit of taxa and distributions failed" + e.getMessage());
160
							e.printStackTrace();
161
						}
162
	
163
						if(logger.isInfoEnabled()) { logger.info("i = " + i + " - Transaction committed");}
164
					}
165
	
166
	
167
				}
168
				if (taxonUuids != null){
169
					try {
170
						commitTaxaAndVernacularNames(state, taxonUuids, fauEuVernacularNamesMap, txStatus);
171
						taxonUuids = null;
172
						taxonList = null;
173
						fauEuVernacularNamesMap = null;
174
					} catch (Exception e) {
175
						logger.error("Commit of taxa and vernacular names failed");
176
						logger.error(e.getMessage());
177
						e.printStackTrace();
178
					}
179
				}
180
				rs = null;
181
			} catch (SQLException e) {
182
				logger.error("SQLException:" +  e);
183
				state.setUnsuccessfull();
184
			}
185
	
186
			if(logger.isInfoEnabled()) { logger.info("End making vernacular names..."); }
187
	
188
			return;
189
		}
190

  
191
	}
192

  
193
	@Override
194
	protected boolean doCheck(FaunaEuropaeaImportState state) {
195
		// TODO Auto-generated method stub
196
		return false;
197
	}
198

  
199
	@Override
200
	protected boolean isIgnore(FaunaEuropaeaImportState state) {
201
		// TODO Auto-generated method stub
202
		return false;
203
	}
204
	private void commitTaxaAndVernacularNames(
205
			FaunaEuropaeaImportState state, 
206
			Set<UUID> taxonUuids,
207
			Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuTaxonMap,
208
			TransactionStatus txStatus) throws Exception {
209
		List<TaxonBase> taxonList = prepareTaxaAndVernacularNames(getTaxonService().find(taxonUuids), fauEuTaxonMap, state);
210

  
211
		getTaxonService().saveOrUpdate(taxonList);
212
		taxonList = null;
213
		taxonUuids = null;
214
		fauEuTaxonMap = null;
215
		commitTransaction(txStatus);
216

  
217
	}
218
	
219
	private List<TaxonBase> prepareTaxaAndVernacularNames(List<TaxonBase> taxonList, Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuTaxonMap,  FaunaEuropaeaImportState state) throws Exception{
220

  
221
		CommonTaxonName verName = null;
222
		
223
		
224
		FaunaEuropaeaVernacularNamesTaxon fauEuHelperTaxon;
225
		UUID taxonUuid;
226
		TaxonDescription taxonDescription;
227
		Taxon taxon;
228
		for (TaxonBase<?> taxonBase : taxonList) {
229

  
230
			if (taxonBase != null) {
231

  
232
				if (taxonBase instanceof Taxon) {
233
					taxon = CdmBase.deproxy(taxonBase, Taxon.class);
234
				} else {
235
					Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
236
					if (syn.getAcceptedTaxa().size()>0){
237
						taxon = syn.getAcceptedTaxa().iterator().next();
238
					}
239
					logger.warn("TaxonBase (" + taxonBase.getId() + " is not of type Taxon but: "
240
							+ taxonBase.getClass().getSimpleName() + " using accepted Taxon for vernacular name");
241
					continue;
242
				}
243

  
244

  
245
				Set<TaxonDescription> descriptionSet = taxon.getDescriptions();
246
				taxonDescription = TaxonDescription.NewInstance();
247
				taxon.addDescription(taxonDescription);
248
				
249

  
250
				taxonUuid = taxonBase.getUuid();
251
				fauEuHelperTaxon= fauEuTaxonMap.get(taxonUuid);
252

  
253
				for (FaunaEuropaeaVernacularName fauEuHelperVernacularName: fauEuHelperTaxon.getVernacularNames()) {
254
					
255
					verName = null;
256
					
257
					/*
258
					 * UUID uuid = state.getTransformer().getLanguageUuid(languageKey);
259
					result = (Language)getTermService().find(uuid);
260
				}
261
				if (result == null){
262
					result = state.getTransformer().getLanguageByKey(languageKey);
263
					if (result == null){
264
						UUID uuidLanguage;
265
						uuidLanguage = state.getTransformer().getLanguageUuid(languageKey);
266
						if (uuidLanguage == null){
267
							logger.warn("Language not defined: " + languageKey)  ;
268
						}
269
						result = getLanguage(state, uuidLanguage, languageKey, languageKey, null);
270
						if (result == null){
271
							logger.warn("Language not defined: " + languageKey)  ;
272
						}
273
					}else if (result.getId() == 0){
274
//						UUID uuidLanguageVoc = UUID.fromString("45ac7043-7f5e-4f37-92f2-3874aaaef2de"); 
275
						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
276
						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, result);
277
//						TermVocabulary<Language> voc = getVocabularyService().find(uuidLanguageVoc);
278
						voc.addTerm(result);
279
						getTermService().saveOrUpdate(result);
280
						state.putLanguage(result);
281
					}
282
					 * 
283
					 */
284
					Language lang = FaunaEuropaeaTransformer.langFK2Language(fauEuHelperVernacularName.getLanguageFk());
285
					if (lang == null && fauEuHelperVernacularName.getLanguageFk() == 83){
286
						lang = Language.ENGLISH();
287
						fauEuHelperVernacularName.setArea("USA");
288
					} 
289
					
290
					if (lang == null){
291
						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
292
						logger.info("languageFk = " + fauEuHelperVernacularName.getLanguageFk());
293
						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, lang);
294
						lang = Language.NewInstance("Dummy", "", "");
295
						voc.addTerm(lang);
296
						lang =(Language)getTermService().save(lang);
297
						FaunaEuropaeaTransformer.languageFK2Language.put(fauEuHelperVernacularName.getLanguageFk(), lang);
298
					}
299
//					if (lang.getId() == 0){
300
//						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
301
//						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, lang);
302
//						voc.addTerm(lang);
303
//						lang =(Language)getTermService().save(lang);
304
//						//FaunaEuropaeaTransformer.languageFK2Language.put(fauEuHelperVernacularName.getLanguageFk(), lang);
305
//					}
306
					
307
					verName = CommonTaxonName.NewInstance(fauEuHelperVernacularName.getVernacularName(), lang);
308
					if (fauEuHelperVernacularName.getArea()!= null && fauEuHelperVernacularName.getArea().equals("USA")){
309
						verName.setArea(NamedArea.NORTH_AMERICA());
310
					}
311
					verName.setCreated(null);
312
					addOriginalSource(verName, null, null, sourceMap.get(fauEuHelperVernacularName.getSource()));
313
					taxonDescription.addElement(verName);
314
					
315
				}
316
			}
317
		}
318
		return taxonList;
319
	}
320
	
321
	private void createSourceMap(FaunaEuropaeaImportState state){
322
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
323
		Source source = fauEuConfig.getSource();
324
		TransactionStatus txStatus = null;
325
		String sourceSelect = "SELECT DISTINCT vernacularNames.VerSource FROM vernacularNames group by vernacularNames.VerSource";
326
		try {
327
			txStatus =startTransaction();
328
			ResultSet rs = source.getResultSet(sourceSelect);
329
			Reference sourceRef;
330
			String title ;
331
			Set<Reference> referncesToSave = new HashSet<Reference>();
332
			while (rs.next()) {
333
				sourceRef = ReferenceFactory.newGeneric();
334
				title = rs.getString("VerSource");
335
				sourceRef.setTitle(title);
336
				referncesToSave.add(sourceRef);
337
				
338
			}
339
			commitReferences(referncesToSave, txStatus);
340

  
341
		}catch (SQLException e) {
342
			logger.error("SQLException:" +  e);
343
			state.setUnsuccessfull();
344
		}
345

  
346
		
347
	}	
348
	private void commitReferences(Collection<Reference> references,  TransactionStatus txStatus) {
349
		Map <UUID, Reference> referenceMap =getReferenceService().save(references);
350
		Iterator<Entry<UUID, Reference>> it = referenceMap.entrySet().iterator();
351
		while (it.hasNext()){
352
			Reference<?> ref = it.next().getValue();
353
			String refID = ref.getTitle();
354
			sourceMap.put(refID, ref);
355
		}
356
		
357
		commitTransaction(txStatus);
358
	}
359
}
1
package eu.etaxonomy.cdm.io.pesi.faunaEuropaea;
2

  
3
import java.sql.ResultSet;
4
import java.sql.SQLException;
5
import java.util.ArrayList;
6
import java.util.Collection;
7
import java.util.HashMap;
8
import java.util.HashSet;
9
import java.util.Iterator;
10
import java.util.List;
11
import java.util.Map;
12
import java.util.Set;
13
import java.util.UUID;
14
import java.util.Map.Entry;
15

  
16
import org.apache.log4j.Logger;
17
import org.springframework.stereotype.Component;
18
import org.springframework.transaction.TransactionStatus;
19

  
20
import eu.etaxonomy.cdm.io.common.Source;
21
import eu.etaxonomy.cdm.jaxb.TDWGNamespacePrefixMapper;
22
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
23
import eu.etaxonomy.cdm.model.common.CdmBase;
24
import eu.etaxonomy.cdm.model.common.Language;
25
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
26
import eu.etaxonomy.cdm.model.common.TermType;
27
import eu.etaxonomy.cdm.model.common.TermVocabulary;
28
import eu.etaxonomy.cdm.model.description.CommonTaxonName;
29
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
30
import eu.etaxonomy.cdm.model.description.Distribution;
31
import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
32
import eu.etaxonomy.cdm.model.description.TaxonDescription;
33
import eu.etaxonomy.cdm.model.location.NamedArea;
34
import eu.etaxonomy.cdm.model.reference.Reference;
35
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
36
import eu.etaxonomy.cdm.model.taxon.Synonym;
37
import eu.etaxonomy.cdm.model.taxon.Taxon;
38
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
39

  
40
@Component
41
public class FaunaEuropaeaVernacularNamesImport extends FaunaEuropaeaImportBase {
42
	private static final Logger logger = Logger.getLogger(FaunaEuropaeaVernacularNamesImport.class);
43

  
44
	private HashMap<String, Reference> sourceMap = new HashMap<String, Reference>();
45
	
46
	@Override
47
	protected void doInvoke(FaunaEuropaeaImportState state) {
48
		int limit = state.getConfig().getLimitSave();
49
		
50
		if (state.getConfig().isDoVernacularNames()){
51
			/* Taxon store for retrieving taxa from and saving taxa to CDM */
52
			List<TaxonBase> taxonList = null;
53
			/* UUID store as input for retrieving taxa from CDM */
54
			Set<UUID> taxonUuids = null;
55
			/* Store to hold helper objects */
56
			Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuVernacularNamesMap = null;
57
	
58
	
59
			TransactionStatus txStatus = null;
60
			
61
			createSourceMap(state);
62
			
63
			FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
64
			Source source = fauEuConfig.getSource();
65
	
66
	        int i = 0;
67
	
68
			String selectCount =
69
				" SELECT count(*) ";
70
	
71
			String selectColumns =
72
				" SELECT vernacularNames.*, Taxon.UUID ";
73
	
74
			String fromClause =
75
				" FROM vernacularNames INNER JOIN " +
76
	            " Taxon ON vernacularnames.UUID = Taxon.UUID ";
77
			String orderBy = " ORDER BY vernacularNames.UUID";
78
	
79
			String countQuery =
80
				selectCount + fromClause;
81
	
82
			String selectQuery =
83
				selectColumns + fromClause + orderBy;
84
	
85
	
86
	
87
			if(logger.isInfoEnabled()) { logger.info("Start making vernacular names..."); }
88
	
89
			try {
90
				ResultSet rs = source.getResultSet(countQuery);
91
				rs.next();
92
				int count = rs.getInt(1);
93
	
94
				rs = source.getResultSet(selectQuery);
95
	
96
				if (logger.isInfoEnabled()) {
97
					logger.info("Number of rows: " + count);
98
					logger.info("Count Query: " + countQuery);
99
					logger.info("Select Query: " + selectQuery);
100
				}
101
	
102
				//int taxonId;
103
	
104
				while (rs.next()) {
105
					if ((i++ % limit) == 0) {
106
	
107
						txStatus = startTransaction();
108
						taxonUuids = new HashSet<UUID>(limit);
109
						fauEuVernacularNamesMap = new HashMap<UUID, FaunaEuropaeaVernacularNamesTaxon>(limit);
110
	
111
						if(logger.isInfoEnabled()) {
112
							logger.info("i = " + i + " - vernacular names import transaction started");
113
						}
114
					}
115
	
116
					//taxonId = rs.getInt("dis_tax_id");
117
					String vernameGUID = rs.getString("GUID");
118
					String vernacularNameString = rs.getString("verName");
119
					//int taxonFk = rs.getInt("taxonFK");
120
					int languageFK= rs.getInt("languageFK");
121
					String languageCache = rs.getString("languageCache");
122
					String taxonUUID= rs.getString("UUID");
123
					String verSource = rs.getString("VerSource");
124
					UUID currentTaxonUuid = null;
125
					
126
	
127
					FaunaEuropaeaVernacularName fauEuVernacularName = new FaunaEuropaeaVernacularName();
128
					fauEuVernacularName.setGuid(vernameGUID);
129
					fauEuVernacularName.setLanguageCache(languageCache);
130
					fauEuVernacularName.setLanguageFk(languageFK);
131
					fauEuVernacularName.setVernacularName(vernacularNameString);
132
					fauEuVernacularName.setSource(verSource);
133
					fauEuVernacularName.setTaxonUuid(UUID.fromString(taxonUUID));
134
					currentTaxonUuid =UUID.fromString(taxonUUID);
135
					if (!taxonUuids.contains(currentTaxonUuid)) {
136
						taxonUuids.add(currentTaxonUuid);
137
						FaunaEuropaeaVernacularNamesTaxon fauEuVernacularNameTaxon =
138
							new FaunaEuropaeaVernacularNamesTaxon(currentTaxonUuid);
139
						fauEuVernacularNamesMap.put(currentTaxonUuid, fauEuVernacularNameTaxon);
140
						fauEuVernacularNameTaxon = null;
141
					} else {
142
						if (logger.isTraceEnabled()) {
143
							logger.trace("Taxon (" + currentTaxonUuid + ") already stored.");
144
							continue;
145
						}
146
					}
147
	
148
					fauEuVernacularNamesMap.get(currentTaxonUuid).addVernacularName(fauEuVernacularName);
149
	
150
					if (((i % limit) == 0 && i != 1 ) || i == count ) {
151
	
152
						try {
153
							commitTaxaAndVernacularNames(state,  taxonUuids, fauEuVernacularNamesMap, txStatus);
154
							taxonUuids = null;
155
							taxonList = null;
156
							fauEuVernacularNamesMap = null;
157
	
158
						} catch (Exception e) {
159
							logger.error("Commit of taxa and distributions failed" + e.getMessage());
160
							e.printStackTrace();
161
						}
162
	
163
						if(logger.isInfoEnabled()) { logger.info("i = " + i + " - Transaction committed");}
164
					}
165
	
166
	
167
				}
168
				if (taxonUuids != null){
169
					try {
170
						commitTaxaAndVernacularNames(state, taxonUuids, fauEuVernacularNamesMap, txStatus);
171
						taxonUuids = null;
172
						taxonList = null;
173
						fauEuVernacularNamesMap = null;
174
					} catch (Exception e) {
175
						logger.error("Commit of taxa and vernacular names failed");
176
						logger.error(e.getMessage());
177
						e.printStackTrace();
178
					}
179
				}
180
				rs = null;
181
			} catch (SQLException e) {
182
				logger.error("SQLException:" +  e);
183
				state.setUnsuccessfull();
184
			}
185
	
186
			if(logger.isInfoEnabled()) { logger.info("End making vernacular names..."); }
187
	
188
			return;
189
		}
190

  
191
	}
192

  
193
	@Override
194
	protected boolean doCheck(FaunaEuropaeaImportState state) {
195
		// TODO Auto-generated method stub
196
		return false;
197
	}
198

  
199
	@Override
200
	protected boolean isIgnore(FaunaEuropaeaImportState state) {
201
		// TODO Auto-generated method stub
202
		return false;
203
	}
204
	private void commitTaxaAndVernacularNames(
205
			FaunaEuropaeaImportState state, 
206
			Set<UUID> taxonUuids,
207
			Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuTaxonMap,
208
			TransactionStatus txStatus) throws Exception {
209
		List<TaxonBase> taxonList = prepareTaxaAndVernacularNames(getTaxonService().find(taxonUuids), fauEuTaxonMap, state);
210

  
211
		getTaxonService().saveOrUpdate(taxonList);
212
		taxonList = null;
213
		taxonUuids = null;
214
		fauEuTaxonMap = null;
215
		commitTransaction(txStatus);
216

  
217
	}
218
	
219
	private List<TaxonBase> prepareTaxaAndVernacularNames(List<TaxonBase> taxonList, Map<UUID, FaunaEuropaeaVernacularNamesTaxon> fauEuTaxonMap,  FaunaEuropaeaImportState state) throws Exception{
220

  
221
		CommonTaxonName verName = null;
222
		
223
		
224
		FaunaEuropaeaVernacularNamesTaxon fauEuHelperTaxon;
225
		UUID taxonUuid;
226
		TaxonDescription taxonDescription;
227
		Taxon taxon;
228
		for (TaxonBase<?> taxonBase : taxonList) {
229

  
230
			if (taxonBase != null) {
231

  
232
				if (taxonBase instanceof Taxon) {
233
					taxon = CdmBase.deproxy(taxonBase, Taxon.class);
234
				} else {
235
					Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
236
					if (syn.getAcceptedTaxa().size()>0){
237
						taxon = syn.getAcceptedTaxa().iterator().next();
238
					}
239
					logger.warn("TaxonBase (" + taxonBase.getId() + " is not of type Taxon but: "
240
							+ taxonBase.getClass().getSimpleName() + " using accepted Taxon for vernacular name");
241
					continue;
242
				}
243

  
244

  
245
				Set<TaxonDescription> descriptionSet = taxon.getDescriptions();
246
				taxonDescription = TaxonDescription.NewInstance();
247
				taxon.addDescription(taxonDescription);
248
				
249

  
250
				taxonUuid = taxonBase.getUuid();
251
				fauEuHelperTaxon= fauEuTaxonMap.get(taxonUuid);
252

  
253
				for (FaunaEuropaeaVernacularName fauEuHelperVernacularName: fauEuHelperTaxon.getVernacularNames()) {
254
					
255
					verName = null;
256
					
257
					/*
258
					 * UUID uuid = state.getTransformer().getLanguageUuid(languageKey);
259
					result = (Language)getTermService().find(uuid);
260
				}
261
				if (result == null){
262
					result = state.getTransformer().getLanguageByKey(languageKey);
263
					if (result == null){
264
						UUID uuidLanguage;
265
						uuidLanguage = state.getTransformer().getLanguageUuid(languageKey);
266
						if (uuidLanguage == null){
267
							logger.warn("Language not defined: " + languageKey)  ;
268
						}
269
						result = getLanguage(state, uuidLanguage, languageKey, languageKey, null);
270
						if (result == null){
271
							logger.warn("Language not defined: " + languageKey)  ;
272
						}
273
					}else if (result.getId() == 0){
274
//						UUID uuidLanguageVoc = UUID.fromString("45ac7043-7f5e-4f37-92f2-3874aaaef2de"); 
275
						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
276
						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, result);
277
//						TermVocabulary<Language> voc = getVocabularyService().find(uuidLanguageVoc);
278
						voc.addTerm(result);
279
						getTermService().saveOrUpdate(result);
280
						state.putLanguage(result);
281
					}
282
					 * 
283
					 */
284
					Language lang = FaunaEuropaeaTransformer.langFK2Language(fauEuHelperVernacularName.getLanguageFk());
285
					if (lang == null && fauEuHelperVernacularName.getLanguageFk() == 83){
286
						lang = Language.ENGLISH();
287
						fauEuHelperVernacularName.setArea("USA");
288
					} 
289
					
290
					if (lang == null){
291
						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
292
						logger.info("languageFk = " + fauEuHelperVernacularName.getLanguageFk());
293
						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, lang);
294
						lang = Language.NewInstance("Dummy", "", "");
295
						voc.addTerm(lang);
296
						lang =(Language)getTermService().save(lang);
297
						FaunaEuropaeaTransformer.languageFK2Language.put(fauEuHelperVernacularName.getLanguageFk(), lang);
298
					}
299
//					if (lang.getId() == 0){
300
//						UUID uuidLanguageVoc = UUID.fromString("434cea89-9052-4567-b2db-ff77f42e9084"); 
301
//						TermVocabulary<Language> voc = getVocabulary(TermType.Language, uuidLanguageVoc, "User Defined Languages", "User Defined Languages", null, null, false, lang);
302
//						voc.addTerm(lang);
303
//						lang =(Language)getTermService().save(lang);
304
//						//FaunaEuropaeaTransformer.languageFK2Language.put(fauEuHelperVernacularName.getLanguageFk(), lang);
305
//					}
306
					
307
					verName = CommonTaxonName.NewInstance(fauEuHelperVernacularName.getVernacularName(), lang);
308
					if (fauEuHelperVernacularName.getArea()!= null && fauEuHelperVernacularName.getArea().equals("USA")){
309
						verName.setArea(NamedArea.NORTH_AMERICA());
310
					}
311
					verName.setCreated(null);
312
					addOriginalSource(verName, null, null, sourceMap.get(fauEuHelperVernacularName.getSource()));
313
					taxonDescription.addElement(verName);
314
					
315
				}
316
			}
317
		}
318
		return taxonList;
319
	}
320
	
321
	private void createSourceMap(FaunaEuropaeaImportState state){
322
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
323
		Source source = fauEuConfig.getSource();
324
		TransactionStatus txStatus = null;
325
		String sourceSelect = "SELECT DISTINCT vernacularNames.VerSource FROM vernacularNames group by vernacularNames.VerSource";
326
		try {
327
			txStatus =startTransaction();
328
			ResultSet rs = source.getResultSet(sourceSelect);
329
			Reference sourceRef;
330
			String title ;
331
			Set<Reference> referncesToSave = new HashSet<Reference>();
332
			while (rs.next()) {
333
				sourceRef = ReferenceFactory.newGeneric();
334
				title = rs.getString("VerSource");
335
				sourceRef.setTitle(title);
336
				referncesToSave.add(sourceRef);
337
				
338
			}
339
			commitReferences(referncesToSave, txStatus);
340

  
341
		}catch (SQLException e) {
342
			logger.error("SQLException:" +  e);
343
			state.setUnsuccessfull();
344
		}
345

  
346
		
347
	}	
348
	private void commitReferences(Collection<Reference> references,  TransactionStatus txStatus) {
349
		Map <UUID, Reference> referenceMap =getReferenceService().save(references);
350
		Iterator<Entry<UUID, Reference>> it = referenceMap.entrySet().iterator();
351
		while (it.hasNext()){
352
			Reference<?> ref = it.next().getValue();
353
			String refID = ref.getTitle();
354
			sourceMap.put(refID, ref);
355
		}
356
		
357
		commitTransaction(txStatus);
358
	}
359
}
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaVernacularNamesTaxon.java
1
package eu.etaxonomy.cdm.io.pesi.faunaEuropaea;
2

  
3
import java.util.HashSet;
4
import java.util.Set;
5
import java.util.UUID;
6

  
7
public class FaunaEuropaeaVernacularNamesTaxon {
8
	
9
	private UUID taxonUuid;
10
	private int taxonId;
11
	private Set<FaunaEuropaeaVernacularName> vernacularNames = new HashSet<FaunaEuropaeaVernacularName>();	
12
	
13
	public FaunaEuropaeaVernacularNamesTaxon(UUID currentTaxonUuid) {
14
		this.taxonUuid = currentTaxonUuid;
15
	}
16

  
17
	public UUID getTaxonUuid() {
18
		return taxonUuid;
19
	}
20

  
21
	public void setTaxonUuid(UUID taxonUuid) {
22
		this.taxonUuid = taxonUuid;
23
	}
24

  
25
	public int getTaxonId() {
26
		return taxonId;
27
	}
28

  
29
	public void setTaxonId(int taxonId) {
30
		this.taxonId = taxonId;
31
	}
32

  
33
	public Set<FaunaEuropaeaVernacularName> getVernacularNames() {
34
		return vernacularNames;
35
	}
36

  
37
	public void setVernacularNames(Set<FaunaEuropaeaVernacularName> vernacularNames) {
38
		this.vernacularNames = vernacularNames;
39
	}
40

  
41
	public void addVernacularName(FaunaEuropaeaVernacularName fauEuVernacularname) {
42
		vernacularNames.add(fauEuVernacularname);
43
	}
44
	
45
}
1
package eu.etaxonomy.cdm.io.pesi.faunaEuropaea;
2

  
3
import java.util.HashSet;
4
import java.util.Set;
5
import java.util.UUID;
6

  
7
public class FaunaEuropaeaVernacularNamesTaxon {
8
	
9
	private UUID taxonUuid;
10
	private int taxonId;
11
	private Set<FaunaEuropaeaVernacularName> vernacularNames = new HashSet<FaunaEuropaeaVernacularName>();	
12
	
13
	public FaunaEuropaeaVernacularNamesTaxon(UUID currentTaxonUuid) {
14
		this.taxonUuid = currentTaxonUuid;
15
	}
16

  
17
	public UUID getTaxonUuid() {
18
		return taxonUuid;
19
	}
20

  
21
	public void setTaxonUuid(UUID taxonUuid) {
22
		this.taxonUuid = taxonUuid;
23
	}
24

  
25
	public int getTaxonId() {
26
		return taxonId;
27
	}
28

  
29
	public void setTaxonId(int taxonId) {
30
		this.taxonId = taxonId;
31
	}
32

  
33
	public Set<FaunaEuropaeaVernacularName> getVernacularNames() {
34
		return vernacularNames;
35
	}
36

  
37
	public void setVernacularNames(Set<FaunaEuropaeaVernacularName> vernacularNames) {
38
		this.vernacularNames = vernacularNames;
39
	}
40

  
41
	public void addVernacularName(FaunaEuropaeaVernacularName fauEuVernacularname) {
42
		vernacularNames.add(fauEuVernacularname);
43
	}
44
	
45
}

Also available in: Unified diff