Project

General

Profile

« Previous | Next » 

Revision 5ba511b1

Added by Cherian Mathew about 9 years ago

Catalogue of Life (i4Life) Dwca import / export activators

View differences:

.gitattributes
39 39
app-import/src/main/java/eu/etaxonomy/cdm/app/excelImport/taxa/UseSummaryActivator.java -text
40 40
app-import/src/main/java/eu/etaxonomy/cdm/app/exel/NormalExplicitTestActivator.java -text
41 41
app-import/src/main/java/eu/etaxonomy/cdm/app/globis/GlobisActivator.java -text
42
app-import/src/main/java/eu/etaxonomy/cdm/app/i4life/col/ColDwcaExportActivator.java -text
43
app-import/src/main/java/eu/etaxonomy/cdm/app/i4life/col/ColDwcaImportActivator.java -text
42 44
app-import/src/main/java/eu/etaxonomy/cdm/app/images/AbstractImageImporter.java -text
43 45
app-import/src/main/java/eu/etaxonomy/cdm/app/images/ImageImportConfigurator.java -text
44 46
app-import/src/main/java/eu/etaxonomy/cdm/app/images/ImageImportState.java -text
app-import/src/main/java/eu/etaxonomy/cdm/app/i4life/col/ColDwcaExportActivator.java
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
4
* http://www.e-taxonomy.eu
5
* 
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

  
10
package eu.etaxonomy.cdm.app.i4life.col;
11

  
12
import java.io.File;
13
import java.net.URI;
14
import java.util.Arrays;
15
import java.util.List;
16
import java.util.UUID;
17

  
18
import org.apache.log4j.Logger;
19
import org.joda.time.DateTime;
20

  
21
import eu.etaxonomy.cdm.app.common.CdmDestinations;
22
import eu.etaxonomy.cdm.common.monitor.DefaultProgressMonitor;
23
import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
24
import eu.etaxonomy.cdm.database.ICdmDataSource;
25
import eu.etaxonomy.cdm.io.common.CdmDefaultExport;
26
import eu.etaxonomy.cdm.io.common.IExportConfigurator.CHECK;
27
import eu.etaxonomy.cdm.io.dwca.out.DwcaEmlRecord;
28
import eu.etaxonomy.cdm.io.dwca.out.DwcaTaxExportConfigurator;
29
import eu.etaxonomy.cdm.model.agent.Institution;
30
import eu.etaxonomy.cdm.model.agent.InstitutionalMembership;
31
import eu.etaxonomy.cdm.model.agent.Person;
32

  
33

  
34
/**
35
 *
36
 * @author a.mueller
37
 *
38
 */
39
public class ColDwcaExportActivator {
40
	@SuppressWarnings("unused")
41
	private static final Logger logger = Logger.getLogger(ColDwcaExportActivator.class);
42

  
43
	//database validation status (create, update, validate ...)
44
	private static final String fileDestination = "C:\\tmp\\dwcaTmp\\cyprus\\";
45
//	private static final ICdmDataSource cdmSource = CdmDestinations.cdm_local_cichorieae();
46
//	private static final ICdmDataSource cdmSource = CdmDestinations.cdm_test_local_mysql();
47
//	private static final ICdmDataSource cdmSource = CdmDestinations.cdm_production_cichorieae();
48
//	private static final ICdmDataSource cdmSource = CdmDestinations.cdm_flora_central_africa_production();
49
	private static final ICdmDataSource cdmSource = CdmDestinations.cdm_cyprus_production();
50

  
51
	private IProgressMonitor monitor = DefaultProgressMonitor.NewInstance();
52
	
53
	private static DateTime dateTime = new DateTime();
54
	private static String date = dateTime.getYear() + "-" + dateTime.getMonthOfYear() + "-" + dateTime.getDayOfMonth();
55
	
56
	private static final String defaultBibliographicCitation = "ICN (Hand, R., Kilian, N. & Raab-Straube, E. von; general editors) 2009+ (continuously updated): International Cichorieae Network: Cichorieae Portal. Published on the Internet at http://wp6-cichorieae.e-taxonomy.eu/portal/; " + 
57
		"accessed ["+date+"].";
58

  
59
	private static final String taxonSourceDefault = "http://wp6-cichorieae.e-taxonomy.eu/portal/?q=cdm_dataportal/taxon/{id}";
60
	
61
	//check - import
62
	private static final CHECK check = CHECK.EXPORT_WITHOUT_CHECK;
63

  
64
	private static List<UUID> featureExclusions = Arrays.asList(new UUID[]{
65
			UUID.fromString("5deff505-1a32-4817-9a74-50e6936fd630"),   //occurrences
66
			UUID.fromString("8075074c-ace8-496b-ac82-47c14553f7fd"),    //Editor_Parenthesis
67
			UUID.fromString("c0cc5ebe-1f0c-4c31-af53-d486858ea415"),   //Image Sources
68
			UUID.fromString("9f6c551d-0f19-45ea-a855-4946f6fc1093"),		//Credits
69
			UUID.fromString("cbf12c6c-94e6-4724-9c48-0f6f10d83e1c"),   //Editor Brackets
70
			UUID.fromString("0508114d-4158-48b5-9100-369fa75120d3")     //inedited
71
	});
72
	
73
	
74
// ****************** ALL *****************************************
75
	
76
//	private boolean doTaxa = true;
77
//	private boolean doResourceRelation = true;
78
//	private boolean doTypesAndSpecimen = true;
79
//	private boolean doVernacularNames = true;
80
//	private boolean doReferences = true;
81
//	private boolean doDescription = true;
82
//	private boolean doDistributions = true;
83
//	private boolean doImages = false;
84
//	private boolean doMetaData = true;
85
//	private boolean doEml = true;
86

  
87
// ************************ NONE **************************************** //
88
	
89
	private boolean doTaxa = true;
90
	private boolean doResourceRelation = false;
91
	private boolean doTypesAndSpecimen = false;
92
	private boolean doVernacularNames = false;
93
	private boolean doReferences = false;
94
	private boolean doDescription = false;
95
	private boolean doDistributions = false;
96
	private boolean doImages = false;
97
	private boolean doMetaData = true;
98
	private boolean doEml = false;
99
	
100
	public boolean 	doExport(ICdmDataSource source){
101
		System.out.println("Start export to DWC-A ("+ fileDestination + ") ...");
102
		
103
//		CdmUpdater su = CdmUpdater.NewInstance();
104
//		IProgressMonitor monitor = DefaultProgressMonitor.NewInstance();
105
//		
106
//		try {
107
//			su.updateToCurrentVersion(source, monitor);
108
//		} catch (Exception e) {
109
//			e.printStackTrace();
110
//		}
111
//		if (true){
112
//			return true;
113
//		}
114
		
115
		//make file destination
116
		String destination = fileDestination;
117
		DwcaEmlRecord emlRecord = getEmlRecord();
118
		
119
		DwcaTaxExportConfigurator config = DwcaTaxExportConfigurator.NewInstance(source, new File(destination), emlRecord);
120
		
121
		config.setDoTaxa(doTaxa);
122
		config.setDoResourceRelation(doResourceRelation);
123
		config.setDoTypesAndSpecimen(doTypesAndSpecimen);
124
		config.setDoVernacularNames(doVernacularNames);
125
		config.setDoReferences(doReferences);
126
		config.setDoDescription(doDescription);
127
		config.setDoDistributions(doDistributions);
128
		config.setDoImages(doImages);
129
		config.setDoMetaData(doMetaData);
130
		config.setDoEml(doEml);
131
		config.setCheck(check);
132
		config.setProgressMonitor(monitor);
133
		config.setDefaultBibliographicCitation(defaultBibliographicCitation);
134
		config.setDefaultTaxonSource(taxonSourceDefault);
135
		monitor.beginTask("DwcA-Export", 10);
136
		
137
		config.setFeatureExclusions(featureExclusions);
138
		// invoke import
139
		CdmDefaultExport<DwcaTaxExportConfigurator> bmExport = new CdmDefaultExport<DwcaTaxExportConfigurator>();
140
		boolean result = bmExport.invoke(config);
141
		
142
		System.out.println("End export to DWC-A ("+ fileDestination + ")..." + (result? "(successful)":"(with errors)"));
143
		return result;
144
	}
145

  
146

  
147
	private DwcaEmlRecord getEmlRecord() {
148
		DwcaEmlRecord emlRecord = new DwcaEmlRecord();
149
		emlRecord.setIdentifier("My Identifier");
150
		emlRecord.setTitle("Flora of cyprus");
151
		emlRecord.setPublicationDate(new DateTime());
152
		emlRecord.setExpectedCitation("Expected Citation");
153
		emlRecord.setAbstractInfo("The abstract");
154
		emlRecord.setAdditionalInformation("Add info");
155
		emlRecord.setResourceLanguage(null);
156
		emlRecord.setResourceUrl(URI.create("http://www.flora-of-cyprus.eu//portal/"));
157
		emlRecord.setMetaDataLanguage(null);
158
		emlRecord.setResourceLogoUri(null);
159
		emlRecord.setCreativeCommonsLicensing(null);
160
		emlRecord.setProjectTitle("Projekt tit");
161
		emlRecord.setProjectLead("Proj Lead");
162
		emlRecord.setProjectDescription("Proj Desc");
163
		
164
		Person person = Person.NewInstance();
165
		Institution institution = Institution.NewInstance();
166
		
167
		InstitutionalMembership m = person.addInstitutionalMembership(institution, null, null, null);
168
		emlRecord.setResourceCreator(m);
169
		
170
		return emlRecord;
171
	}
172

  
173

  
174
	/**
175
	 * @param args
176
	 */
177
	public static void main(String[] args) {
178
		ColDwcaExportActivator ex = new ColDwcaExportActivator();
179
		ICdmDataSource source = CdmDestinations.chooseDestination(args) != null ? CdmDestinations.chooseDestination(args) : cdmSource;
180

  
181
		ex.doExport(source);
182
	}
183
	
184
	
185
	
186

  
187
}
app-import/src/main/java/eu/etaxonomy/cdm/app/i4life/col/ColDwcaImportActivator.java
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
4
* http://www.e-taxonomy.eu
5
* 
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

  
10
package eu.etaxonomy.cdm.app.i4life.col;
11

  
12
import java.net.URI;
13
import java.util.UUID;
14

  
15
import org.apache.log4j.Logger;
16

  
17
import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
18
import eu.etaxonomy.cdm.app.common.CdmDestinations;
19
import eu.etaxonomy.cdm.database.DbSchemaValidation;
20
import eu.etaxonomy.cdm.database.ICdmDataSource;
21
import eu.etaxonomy.cdm.io.common.CdmDefaultImport;
22
import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
23
import eu.etaxonomy.cdm.io.common.events.LoggingIoObserver;
24
import eu.etaxonomy.cdm.io.dwca.in.DwcaImportConfigurator;
25
import eu.etaxonomy.cdm.io.dwca.in.DwcaImportConfigurator.DatasetUse;
26
import eu.etaxonomy.cdm.io.dwca.in.IImportMapping.MappingType;
27
import eu.etaxonomy.cdm.model.agent.Person;
28
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
29
import eu.etaxonomy.cdm.model.reference.Reference;
30
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
31

  
32
/**
33
 * @author a.mueller
34
 * @created 10.11.2011
35
 */
36
public class ColDwcaImportActivator {
37
	private static final Logger logger = Logger.getLogger(ColDwcaImportActivator.class);
38
	
39
	//database validation status (create, update, validate ...)
40
	static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
41
	static final URI source = dwca_col_All();
42

  
43
	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql_dwca();
44

  
45

  
46
	static boolean isNoQuotes = true;
47
	
48
	//classification
49
	static final UUID classificationUuid = UUID.fromString("29d4011f-a6dd-4081-beb8-559ba6b84a6b");
50
	
51
	//default nom code is ICZN as it allows adding publication year 
52
	static final NomenclaturalCode defaultNomCode = NomenclaturalCode.ICZN;
53

  
54
	
55
	//check - import
56
	static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
57
	static int partitionSize = 1000;
58
	
59
	//config
60
	static DatasetUse datasetUse = DatasetUse.ORIGINAL_SOURCE;
61
	static boolean scientificNameIdAsOriginalSourceId = true;
62
	static boolean guessNomRef = true;
63
	private boolean handleAllRefsAsCitation = true;
64
	
65
	//validate
66
	static boolean validateRankConsistency = false;
67
	
68
	
69
	//taxa
70
	static final boolean doTaxa = true;
71
	static final boolean doDistribution = false;
72
	//deduplicate
73
	static final boolean doDeduplicate = false;
74
	
75
	
76
	
77
	static final MappingType mappingType = MappingType.InMemoryMapping;
78
	
79
	private void doImport(ICdmDataSource cdmDestination){
80
		
81
		//make Source
82
		DwcaImportConfigurator config= DwcaImportConfigurator.NewInstance(source, cdmDestination);
83
		config.addObserver(new LoggingIoObserver());
84
		config.setClassificationUuid(classificationUuid);
85
		config.setCheck(check);
86
		config.setDbSchemaValidation(hbm2dll);
87
		config.setMappingType(mappingType);
88
		
89
		config.setScientificNameIdAsOriginalSourceId(scientificNameIdAsOriginalSourceId);
90
		config.setValidateRankConsistency(validateRankConsistency);
91
		config.setDefaultPartitionSize(partitionSize);
92
		config.setNomenclaturalCode(defaultNomCode);
93
		config.setDatasetUse(datasetUse);
94
		config.setGuessNomenclaturalReferences(guessNomRef);
95
		config.setHandleAllRefsAsCitation(handleAllRefsAsCitation);
96
		config.setNoQuotes(isNoQuotes);
97
		
98
		CdmDefaultImport myImport = new CdmDefaultImport();
99

  
100
		
101
		//...
102
		if (true){
103
			System.out.println("Start import from ("+ source.toString() + ") ...");
104
			config.setSourceReference(getSourceReference(config.getSourceReferenceTitle()));
105
			myImport.invoke(config);
106
			System.out.println("End import from ("+ source.toString() + ")...");
107
		}
108
		
109
		
110
		
111
		//deduplicate
112
		if (doDeduplicate){
113
			ICdmApplicationConfiguration app = myImport.getCdmAppController();
114
			int count = app.getAgentService().deduplicate(Person.class, null, null);
115
			logger.warn("Deduplicated " + count + " persons.");
116
//			count = app.getAgentService().deduplicate(Team.class, null, null);
117
//			logger.warn("Deduplicated " + count + " teams.");
118
			count = app.getReferenceService().deduplicate(Reference.class, null, null);
119
			logger.warn("Deduplicated " + count + " references.");
120
		}
121
		
122
	}
123

  
124
	private Reference<?> getSourceReference(String string) {
125
		Reference<?> result = ReferenceFactory.newGeneric();
126
		result.setTitleCache(string);
127
		return result;
128
	}
129

  
130
	//Dwca
131
	public static URI dwca_test_in() {
132
//		URI sourceUrl = URI.create("http://dev.e-taxonomy.eu/trac/export/14463/trunk/cdmlib/cdmlib-io/src/test/resources/eu/etaxonomy/cdm/io/dwca/in/DwcaZipToStreamConverterTest-input.zip");
133
		URI sourceUrl = URI.create("file:///C:/Users/pesiimport/Documents/pesi_cdmlib/cdmlib-io/src/test/resources/eu/etaxonomy/cdm/io/dwca/in/DwcaZipToStreamConverterTest-input.zip");
134
		return sourceUrl;
135
	}
136
	
137
	
138
	//CoL
139
	public static URI dwca_col_All() {
140
		URI sourceUrl = URI.create("file:////Pesiimport3/col/col_20Nov2012.zip");
141
		return sourceUrl;
142
	}
143

  
144
	/**
145
	 * @param args
146
	 */
147
	public static void main(String[] args) {
148
		ColDwcaImportActivator me = new ColDwcaImportActivator();
149
		me.doImport(cdmDestination);
150
	}
151
	
152
}

Also available in: Unified diff