Project

General

Profile

« Previous | Next » 

Revision 512f31a5

Added by Andreas Müller about 12 years ago

updates to Vibrant activators

View differences:

.gitattributes
69 69
app-import/src/main/java/eu/etaxonomy/cdm/app/tcs/TcsXmlTestActivator.java -text
70 70
app-import/src/main/java/eu/etaxonomy/cdm/app/util/TestDatabase.java -text
71 71
app-import/src/main/java/eu/etaxonomy/cdm/app/util/TestTransaction.java -text
72
app-import/src/main/java/eu/etaxonomy/cdm/app/vibrant/EuroMedActivator.java -text
72 73
app-import/src/main/java/eu/etaxonomy/cdm/app/vibrant/IopiActivator.java -text
73 74
app-import/src/main/java/eu/etaxonomy/cdm/app/vibrant/MclActivator.java -text
74 75
app-import/src/main/java/eu/etaxonomy/cdm/app/vibrant/VibrantActivator.java -text
app-import/src/main/java/eu/etaxonomy/cdm/app/vibrant/EuroMedActivator.java
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
4
* http://www.e-taxonomy.eu
5
* 
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

  
10
package eu.etaxonomy.cdm.app.vibrant;
11

  
12
import java.lang.reflect.Method;
13
import java.sql.ResultSet;
14
import java.sql.SQLException;
15
import java.util.UUID;
16

  
17
import org.apache.log4j.Logger;
18

  
19
import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
20
import eu.etaxonomy.cdm.app.common.BerlinModelSources;
21
import eu.etaxonomy.cdm.app.common.CdmDestinations;
22
import eu.etaxonomy.cdm.app.common.TreeCreator;
23
import eu.etaxonomy.cdm.database.DbSchemaValidation;
24
import eu.etaxonomy.cdm.database.ICdmDataSource;
25
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
26
import eu.etaxonomy.cdm.io.common.CdmDefaultImport;
27
import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
28
import eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES;
29
import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;
30
import eu.etaxonomy.cdm.io.common.Source;
31
import eu.etaxonomy.cdm.io.pesi.out.PesiTransformer;
32
import eu.etaxonomy.cdm.model.common.ExtensionType;
33
import eu.etaxonomy.cdm.model.description.Feature;
34
import eu.etaxonomy.cdm.model.description.FeatureNode;
35
import eu.etaxonomy.cdm.model.description.FeatureTree;
36
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
37
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
38

  
39

  
40
/**
41
 * TODO add the following to a wiki page:
42
 * HINT: If you are about to import into a mysql data base running under windows and if you wish to dump and restore the resulting data bas under another operation systen 
43
 * you must set the mysql system variable lower_case_table_names = 0 in order to create data base with table compatible names.
44
 * 
45
 * 
46
 * @author a.mueller
47
 *
48
 */
49
public class EuroMedActivator {
50
	private static final Logger logger = Logger.getLogger(EuroMedActivator.class);
51

  
52
	//database validation status (create, update, validate ...)
53
	static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
54
//	static final Source berlinModelSource = BerlinModelSources.euroMed();
55
	static final Source berlinModelSource = BerlinModelSources.PESI3_euroMed();
56
	
57
//	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_pesi_euroMed();
58
//	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql();
59
	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql();
60
	
61
	static final boolean includePesiExport = false;
62
	
63
	static final int sourceSecId = 7000000; //500000
64
	static final UUID classificationUuid = UUID.fromString("5e05ebc5-6075-45ff-81df-4cefafafa4a3");
65
	static final boolean useSingleClassification = true;
66
	
67
	static final UUID featureTreeUuid = UUID.fromString("eff345e7-0619-4ec3-955d-997c1fafffc3");
68
	static final Object[] featureKeyList = new Integer[]{1, 31, 4, 98, 41}; 	
69
	
70
	// set to zero for unlimited nameFacts
71
	static final int maximumNumberOfNameFacts = 0;
72
	
73
	static final int partitionSize = 2500;
74
	
75
	//check - import
76
	static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
77

  
78
	//editor - import
79
	static final EDITOR editor = EDITOR.EDITOR_AS_EDITOR;
80
	
81
	//NomeclaturalCode
82
	static final NomenclaturalCode nomenclaturalCode = NomenclaturalCode.ICBN;
83

  
84
	//ignore null
85
	static final boolean ignoreNull = true;
86
	
87
	static final boolean switchSpeciesGroup = true;
88
	
89
	static boolean useClassification = true;
90
	
91
	static boolean 	isSplitTdwgCodes = false;
92
	
93
	static String taxonTable = "v_cdm_exp_taxaAll";
94
	static String classificationQuery = " SELECT DISTINCT t.PTRefFk, r.RefCache FROM PTaxon t INNER JOIN Reference r ON t.PTRefFk = r.RefId WHERE t.PTRefFk = " + sourceSecId; 
95
	static String relPTaxonIdQuery = " SELECT r.RelPTaxonId " + 
96
					" FROM RelPTaxon AS r INNER JOIN v_cdm_exp_taxaDirect AS a ON r.PTNameFk2 = a.PTNameFk AND r.PTRefFk2 = a.PTRefFk ";
97
	static String nameIdTable = " v_cdm_exp_namesAll ";
98
	static String referenceIdTable = " v_cdm_exp_refAll ";
99
	static String factFilter = " factId IN ( SELECT factId FROM v_cdm_exp_factsAll )";
100
	static String occurrenceFilter = " occurrenceId IN ( SELECT occurrenceId FROM v_cdm_exp_occurrenceAll )";
101
	static String occurrenceSourceFilter = " occurrenceFk IN ( SELECT occurrenceId FROM v_cdm_exp_occurrenceAll )"; 
102
	static String commonNameFilter = " commonNameId IN ( SELECT commonNameId FROM v_cdm_exp_commonNamesAll )";
103
	static String webMarkerFilter = " TableNameFk <> 500 OR ( RIdentifierFk IN (SELECT RIdentifier FROM v_cdm_exp_taxaAll)) ";
104
	static String authorTeamFilter = null; // " authorTeamId IN (SELECT authorTeamId FROM v_cdm_exp_authorTeamsAll) ";
105
	static String authorFilter = null;  // " authorId IN (SELECT authorId FROM v_cdm_exp_authorsAll) "; 
106
	
107

  
108
	
109
// **************** ALL *********************	
110

  
111
	static final boolean doUser = true;
112
	//authors
113
	static final boolean doAuthors = true;
114
	//references
115
	static final DO_REFERENCES doReferences =  DO_REFERENCES.ALL;
116
	//names
117
	static final boolean doTaxonNames = true;
118
	static final boolean doRelNames = true;
119
	static final boolean doNameStatus = true;
120
	static final boolean doTypes = false;  //serious types do not exist in E+M
121
	static final boolean doNameFacts = true;
122
	
123
	//taxa
124
	static final boolean doTaxa = true;
125
	static final boolean doRelTaxa = true;
126
	static final boolean doFacts = true;
127
	static final boolean doOccurences = true;
128
	static final boolean doCommonNames = true;
129

  
130
	//etc.
131
	static final boolean doMarker = true;
132

  
133
	
134
// **************** SELECTED *********************
135

  
136
//	static final boolean doUser = true;
137
//	//authors
138
//	static final boolean doAuthors = false;
139
//	//references
140
//	static final DO_REFERENCES doReferences =  DO_REFERENCES.NONE;
141
//	//names
142
//	static final boolean doTaxonNames = false;
143
//	static final boolean doRelNames = false;
144
//	static final boolean doNameStatus = false;
145
//	static final boolean doTypes = false;
146
//	static final boolean doNameFacts = false;
147
//	
148
//	//taxa 
149
//	static final boolean doTaxa = false;
150
//	static final boolean doRelTaxa = false;
151
//	static final boolean doFacts = false;
152
//	static final boolean doOccurences = false;
153
//	static final boolean doCommonNames = false;
154
//	
155
//	//etc.
156
//	static final boolean doMarker = false;
157
	
158
	
159
	public void importEm2CDM (Source source, ICdmDataSource destination, DbSchemaValidation hbm2dll){
160
		System.out.println("Start import from BerlinModel("+ source.getDatabase() + ") to " + destination.getDatabase() + " ...");
161
		//make BerlinModel Source
162
				
163
		BerlinModelImportConfigurator config = BerlinModelImportConfigurator.NewInstance(source,  destination);
164
		
165
		config.setClassificationUuid(classificationUuid);
166
		config.setSourceSecId(sourceSecId);
167
		
168
		config.setNomenclaturalCode(nomenclaturalCode);
169

  
170
		try {
171
			Method makeUrlMethod = MclActivator.class.getDeclaredMethod("makeUrlForTaxon", TaxonBase.class, ResultSet.class);
172
			config.setMakeUrlForTaxon(makeUrlMethod);
173
		} catch (Exception e) {
174
			e.printStackTrace();
175
			return;
176
		}
177
		
178
		
179
		config.setIgnoreNull(ignoreNull);
180
		config.setDoAuthors(doAuthors);
181
		config.setDoReferences(doReferences);
182
		config.setDoTaxonNames(doTaxonNames);
183
		config.setDoRelNames(doRelNames);
184
		config.setDoNameStatus(doNameStatus);
185
		config.setDoTypes(doTypes);
186
		config.setDoNameFacts(doNameFacts);
187
		config.setUseClassification(useClassification);
188
		config.setSourceRefUuid(PesiTransformer.uuidSourceRefEuroMed);
189
		
190
		config.setDoTaxa(doTaxa);
191
		config.setDoRelTaxa(doRelTaxa);
192
		config.setDoFacts(doFacts);
193
		config.setDoOccurrence(doOccurences);
194
		config.setDoCommonNames(doCommonNames);
195
		
196
		config.setDoMarker(doMarker);
197
		config.setDoUser(doUser);
198
		config.setEditor(editor);
199
		config.setDbSchemaValidation(hbm2dll);
200
		
201
		// maximum number of name facts to import
202
		config.setMaximumNumberOfNameFacts(maximumNumberOfNameFacts);
203
		
204
//		filter
205
		config.setTaxonTable(taxonTable);
206
		config.setClassificationQuery(classificationQuery);
207
		config.setRelTaxaIdQuery(relPTaxonIdQuery);
208
		config.setNameIdTable(nameIdTable);
209
		config.setReferenceIdTable(referenceIdTable);
210
		config.setAuthorTeamFilter(authorTeamFilter);
211
		config.setAuthorFilter(authorFilter);
212
		config.setFactFilter(factFilter);
213
		config.setCommonNameFilter(commonNameFilter);
214
		config.setOccurrenceFilter(occurrenceFilter);
215
		config.setOccurrenceSourceFilter(occurrenceSourceFilter);
216
		config.setWebMarkerFilter(webMarkerFilter);
217
		config.setUseSingleClassification(useSingleClassification);
218
		
219
		//TDWG codes
220
		config.setSplitTdwgCodes(isSplitTdwgCodes);
221
		
222
		
223
		config.setCheck(check);
224
		config.setEditor(editor);
225
		config.setRecordsPerTransaction(partitionSize);
226
		
227
		config.setSwitchSpeciesGroup(switchSpeciesGroup);
228
		
229
		// invoke import
230
		CdmDefaultImport<BerlinModelImportConfigurator> bmImport = new CdmDefaultImport<BerlinModelImportConfigurator>();
231
		bmImport.invoke(config);
232
		
233
		if (doFacts && config.getCheck().equals(CHECK.CHECK_AND_IMPORT)  || config.getCheck().equals(CHECK.IMPORT_WITHOUT_CHECK)    ){
234
			ICdmApplicationConfiguration app = bmImport.getCdmAppController();
235
			
236
			//make feature tree
237
			FeatureTree tree = TreeCreator.flatTree(featureTreeUuid, config.getFeatureMap(), featureKeyList);
238
			FeatureNode imageNode = FeatureNode.NewInstance(Feature.IMAGE());
239
			tree.getRoot().addChild(imageNode);
240
			FeatureNode distributionNode = FeatureNode.NewInstance(Feature.DISTRIBUTION());
241
			tree.getRoot().addChild(distributionNode, 2); 
242
			app.getFeatureTreeService().saveOrUpdate(tree);
243
		}
244
		
245
		System.out.println("End import from BerlinModel ("+ source.getDatabase() + ")...");
246
		
247
	}
248
	
249
	/**
250
	 * @param args
251
	 */
252
	public static void main(String[] args) {
253
		EuroMedActivator importActivator = new EuroMedActivator();
254
		Source source = berlinModelSource;
255
		ICdmDataSource cdmRepository = CdmDestinations.chooseDestination(args) != null ? CdmDestinations.chooseDestination(args) : cdmDestination;
256
		
257
		importActivator.importEm2CDM(source, cdmRepository, hbm2dll);
258
	}
259
	
260
	private static final String URLbase = "http://ww2.bgbm.org/EuroPlusMed/PTaxonDetail.asp?";
261
	public static Method makeUrlForTaxon(TaxonBase<?> taxon, ResultSet rs){
262
		Method result = null;
263
		ExtensionType urlExtensionType = ExtensionType.URL();
264
		int nameFk;
265
		try {
266
			nameFk = rs.getInt("PTNameFk");
267
		int refFkInt = rs.getInt("PTRefFk");
268
			if (nameFk != 0 && refFkInt != 0){
269
				String url = String.format(URLbase + "NameId=%s&PTRefFk=%s",nameFk, refFkInt);
270
				taxon.addExtension(url, urlExtensionType);
271
			}else{
272
				logger.warn("NameFk or refFkInt is 0. Can't create url");
273
			}
274
		} catch (SQLException e) {
275
			logger.warn("Exception when trying to access result set for url creation.");
276
		}
277
		
278
		return result;
279
	}
280

  
281
}
app-import/src/main/java/eu/etaxonomy/cdm/app/vibrant/MclActivator.java
9 9

  
10 10
package eu.etaxonomy.cdm.app.vibrant;
11 11

  
12
import java.lang.reflect.Method;
13
import java.sql.ResultSet;
14
import java.sql.SQLException;
12 15
import java.util.UUID;
13 16

  
14 17
import org.apache.log4j.Logger;
......
24 27
import eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES;
25 28
import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;
26 29
import eu.etaxonomy.cdm.io.common.Source;
27
import eu.etaxonomy.cdm.io.pesi.out.PesiTransformer;
30
import eu.etaxonomy.cdm.model.common.ExtensionType;
28 31
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
32
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
29 33

  
30 34

  
31 35
/**
......
38 42
 *
39 43
 */
40 44
public class MclActivator {
41
	@SuppressWarnings("unused")
42 45
	private static final Logger logger = Logger.getLogger(MclActivator.class);
43 46

  
44 47
	//database validation status (create, update, validate ...)
45 48
	static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
46 49
	static final Source berlinModelSource = BerlinModelSources.mcl();
47 50
	
48
	static final ICdmDataSource cdmDestination = cdm_test_local_mcl();
51
	static final ICdmDataSource cdmDestination = VibrantActivator.cdm_test_local_vibrant();
49 52
//	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql();
50 53
	
51 54
	static final boolean useSingleClassification = true;
......
128 131
				
129 132
		BerlinModelImportConfigurator config = BerlinModelImportConfigurator.NewInstance(source,  destination);
130 133
		
134
		try {
135
			Method makeUrlMethod = MclActivator.class.getDeclaredMethod("makeUrlForTaxon", TaxonBase.class, ResultSet.class);
136
			config.setMakeUrlForTaxon(makeUrlMethod);
137
		} catch (Exception e) {
138
			e.printStackTrace();
139
			return;
140
		}
141

  
142
		
131 143
		config.setClassificationUuid(classificationUuid);
132 144
		config.setSourceSecId(sourceSecId);
133 145
		
......
193 205
		String cdmUserName = "root";
194 206
		return CdmDestinations.makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null);
195 207
	}
208
	
209
	private static final String URLbase = "http://ww2.bgbm.org/mcl/PTaxonDetail.asp?";
210
	public static Method makeUrlForTaxon(TaxonBase<?> taxon, ResultSet rs){
211
		Method result = null;
212
		ExtensionType urlExtensionType = ExtensionType.URL();
213
		int nameFk;
214
		try {
215
			nameFk = rs.getInt("PTNameFk");
216
		int refFkInt = rs.getInt("PTRefFk");
217
			if (nameFk != 0 && refFkInt != 0){
218
				String url = String.format(URLbase + "NameId=%s&PTRefFk=%s",nameFk, refFkInt);
219
				taxon.addExtension(url, urlExtensionType);
220
			}else{
221
				logger.warn("NameFk or refFkInt is 0. Can't create url");
222
			}
223
		} catch (SQLException e) {
224
			logger.warn("Exception when trying to access result set for url creation.");
225
		}
226
		
227
		return result;
228
	}
196 229

  
197 230
}
app-import/src/main/java/eu/etaxonomy/cdm/app/vibrant/VibrantActivator.java
9 9

  
10 10
package eu.etaxonomy.cdm.app.vibrant;
11 11

  
12
import java.util.UUID;
13

  
14 12
import org.apache.log4j.Logger;
15 13

  
16 14
import eu.etaxonomy.cdm.app.common.BerlinModelSources;
17 15
import eu.etaxonomy.cdm.app.common.CdmDestinations;
18
import eu.etaxonomy.cdm.app.pesi.EuroMedActivator;
19 16
import eu.etaxonomy.cdm.database.DatabaseTypeEnum;
20 17
import eu.etaxonomy.cdm.database.DbSchemaValidation;
21 18
import eu.etaxonomy.cdm.database.ICdmDataSource;
22
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
23
import eu.etaxonomy.cdm.io.common.CdmDefaultImport;
24
import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
25
import eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES;
26
import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;
27 19
import eu.etaxonomy.cdm.io.common.Source;
28
import eu.etaxonomy.cdm.io.pesi.out.PesiTransformer;
29
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
30 20

  
31 21

  
32 22
/**
......
43 33
	private static final Logger logger = Logger.getLogger(VibrantActivator.class);
44 34

  
45 35
	//database validation status (create, update, validate ...)
46
	static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
36
	static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE;
47 37
	static final Source iopiSource = BerlinModelSources.iopi();
48 38
	static final Source mclSource = BerlinModelSources.mcl();
49 39
	static final Source emSource = BerlinModelSources.euroMed_local();
......
51 41
//	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql();
52 42
	static final ICdmDataSource cdmDestination = cdm_test_local_vibrant();
53 43

  
54
	static final boolean doMcl = true;
55
	static final boolean doEuroMed = false;
56
	static final boolean doIopi = true;
44
	static final boolean doMcl = false;
45
	static final boolean doEuroMed = true;
46
	static final boolean doIopi = false;
57 47
	
58 48
	
59 49
	/**

Also available in: Unified diff