Project

General

Profile

« Previous | Next » 

Revision cc8a23e7

Added by Andreas Müller almost 10 years ago

update for globis import

View differences:

.gitattributes
120 120
app-import/src/main/java/eu/etaxonomy/cdm/io/cyprus/CyprusRow.java -text
121 121
app-import/src/main/java/eu/etaxonomy/cdm/io/cyprus/CyprusTransformer.java -text
122 122
app-import/src/main/java/eu/etaxonomy/cdm/io/cyprus/CyprusUserImport.java -text
123
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisCurrentSpeciesImport.java -text
124
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImageImport.java -text
123 125
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportBase.java -text
124 126
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportConfigurator.java -text
125 127
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportState.java -text
126 128
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisReferenceImport.java -text
129
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisReferenceImport_Mapping_OLD.java -text
130
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisSpecTaxImport.java -text
127 131
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisTransformer.java -text
132
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/validation/GlobisCurrentSpeciesImportValidator.java -text
128 133
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/validation/GlobisReferenceImportValidator.java -text
134
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/validation/GlobisSpecTaxaImportValidator.java -text
129 135
app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/RoteListeDbImportBase.java -text
130 136
app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/RoteListeDbImportConfigurator.java -text
131 137
app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/RoteListeDbImportState.java -text
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisCurrentSpeciesImport.java
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
4
* http://www.e-taxonomy.eu
5
* 
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

  
10
package eu.etaxonomy.cdm.io.globis;
11

  
12
import java.sql.ResultSet;
13
import java.sql.SQLException;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.Map;
17
import java.util.Set;
18

  
19
import org.apache.commons.lang.StringUtils;
20
import org.apache.log4j.Logger;
21
import org.springframework.stereotype.Component;
22

  
23
import com.yourkit.util.Strings;
24

  
25
import eu.etaxonomy.cdm.common.CdmUtils;
26
import eu.etaxonomy.cdm.io.algaterra.AlgaTerraCollectionImport;
27
import eu.etaxonomy.cdm.io.algaterra.AlgaTerraSpecimenImport;
28
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonImport;
29
import eu.etaxonomy.cdm.io.common.IOValidator;
30
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
31
import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
32
import eu.etaxonomy.cdm.io.globis.validation.GlobisCurrentSpeciesImportValidator;
33
import eu.etaxonomy.cdm.model.common.CdmBase;
34
import eu.etaxonomy.cdm.model.common.DefinedTermBase;
35
import eu.etaxonomy.cdm.model.common.Language;
36
import eu.etaxonomy.cdm.model.description.CommonTaxonName;
37
import eu.etaxonomy.cdm.model.description.Distribution;
38
import eu.etaxonomy.cdm.model.description.PresenceTerm;
39
import eu.etaxonomy.cdm.model.description.TaxonDescription;
40
import eu.etaxonomy.cdm.model.location.NamedArea;
41
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
42
import eu.etaxonomy.cdm.model.name.Rank;
43
import eu.etaxonomy.cdm.model.name.ZoologicalName;
44
import eu.etaxonomy.cdm.model.occurrence.Collection;
45
import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
46
import eu.etaxonomy.cdm.model.reference.Reference;
47
import eu.etaxonomy.cdm.model.taxon.Classification;
48
import eu.etaxonomy.cdm.model.taxon.Taxon;
49
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
50
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
51
import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
52
import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
53
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
54

  
55

  
56
/**
57
 * @author a.mueller
58
 * @created 20.02.2010
59
 * @version 1.0
60
 */
61
@Component
62
public class GlobisCurrentSpeciesImport  extends GlobisImportBase<Taxon> {
63
	private static final Logger logger = Logger.getLogger(GlobisCurrentSpeciesImport.class);
64
	
65
	private int modCount = 10000;
66
	private static final String pluralString = "current taxa";
67
	private static final String dbTableName = "current_species";
68
	private static final Class cdmTargetClass = Taxon.class;  //not needed
69
	
70
	public GlobisCurrentSpeciesImport(){
71
		super(pluralString, dbTableName, cdmTargetClass);
72
	}
73

  
74

  
75
	
76
	
77
	/* (non-Javadoc)
78
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#getIdQuery()
79
	 */
80
	@Override
81
	protected String getIdQuery() {
82
		String strRecordQuery = 
83
			" SELECT IDcurrentspec " + 
84
			" FROM " + dbTableName; 
85
		return strRecordQuery;	
86
	}
87

  
88

  
89

  
90

  
91
	/* (non-Javadoc)
92
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
93
	 */
94
	@Override
95
	protected String getRecordQuery(GlobisImportConfigurator config) {
96
		String strRecordQuery = 
97
			" SELECT cs.*, cs.dtSpcEingabedatum as Created_When, cs.dtSpcErfasser as Created_Who," +
98
				"  cs.dtSpcBearbeiter as Updated_who, cs.dtSpcAendrgdatum as Updated_When, cs.dtSpcBemerkung as Notes " + 
99
			" FROM " + getTableName() + " cs " +
100
			" WHERE ( cs.IDcurrentspec IN (" + ID_LIST_TOKEN + ") )";
101
		return strRecordQuery;
102
	}
103
	
104

  
105

  
106
	/* (non-Javadoc)
107
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#doPartition(eu.etaxonomy.cdm.io.common.ResultSetPartitioner, eu.etaxonomy.cdm.io.globis.GlobisImportState)
108
	 */
109
	@Override
110
	public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) {
111
		boolean success = true;
112
		
113
		Set<TaxonBase> objectsToSave = new HashSet<TaxonBase>();
114
		
115
		Map<String, Taxon> taxonMap = (Map<String, Taxon>) partitioner.getObjectMap(TAXON_NAMESPACE);
116
//		Map<String, DerivedUnit> ecoFactDerivedUnitMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(ECO_FACT_DERIVED_UNIT_NAMESPACE);
117
		
118
		ResultSet rs = partitioner.getResultSet();
119

  
120
		Classification classification = getClassification(state);
121
		
122
		try {
123
			
124
			int i = 0;
125

  
126
			//for each reference
127
            while (rs.next()){
128
                
129
        		if ((i++ % modCount) == 0 && i!= 1 ){ logger.info(pluralString + " handled: " + (i-1));}
130
				
131
        		Integer taxonId = rs.getInt("IDcurrentspec");
132
        		
133
        		
134
        		//String dtSpcJahr -> ignore !
135
        		//empty: fiSpcLiteratur
136
        		
137
        		//TODO
138
        		//fiSpcspcgrptax
139
        		
140
        	
141
        		
142
				try {
143
					
144
					//source ref
145
					Reference<?> sourceRef = state.getTransactionalSourceReference();
146
					Taxon nextHigherTaxon = null;
147
					
148
					boolean hasNewParent = false; //true if any parent is new
149
					
150
					//species
151
					Taxon species = createObject(rs, state);
152
					
153
					
154
					String familyStr = rs.getString("dtSpcFamakt");
155
					String subFamilyStr = rs.getString("dtSpcSubfamakt");
156
					String tribeStr = rs.getString("dtSpcTribakt");
157
					
158
					//family
159
					Taxon family = getTaxon(state, rs, familyStr, null, Rank.FAMILY(), null, taxonMap);
160
					
161
					//subfamily
162
					Taxon subFamily = getTaxon(state, rs, subFamilyStr, null, Rank.SUBFAMILY(), null, taxonMap);
163
					Taxon subFamilyParent = getParent(subFamily, classification);
164
					if (subFamilyParent != null){
165
						if (! compareTaxa(family, subFamilyParent)){
166
							logger.warn("Current family and parent of subfamily are not equal: " + taxonId);
167
						}
168
					}else{
169
						classification.addParentChild(family, subFamily, sourceRef, null);
170
					}
171
					nextHigherTaxon = subFamily;
172
					
173
					//tribe
174
					Taxon tribe = getTaxon(state, rs, tribeStr, null, Rank.TRIBE(), null, taxonMap);
175
					if (tribe != null){
176
						Taxon tribeParent = getParent(tribe, classification);
177
						if (tribeParent != null){
178
							if (! compareTaxa(subFamily, tribeParent)){
179
								logger.warn("Current subFamily and parent of tribe are not equal: " + taxonId);
180
							}
181
						}else{
182
							classification.addParentChild(subFamily, tribe, sourceRef, null);
183
						}
184
						nextHigherTaxon = tribe;
185
					}					
186

  
187
					
188
					//genus
189
					String genusStr = rs.getString("dtSpcGenusakt");
190
					String genusAuthorStr = rs.getString("dtSpcGenusaktauthor");
191
					Taxon genus = getTaxon(state, rs, genusStr, null, Rank.GENUS(), genusAuthorStr, taxonMap);
192
					Taxon genusParent = getParent(genus, classification);
193
					
194
					if (genusParent != null){
195
						if (! compareTaxa(genusParent, nextHigherTaxon)){
196
							logger.warn("Current tribe/subfamily and parent of genus are not equal: " + taxonId);
197
						}
198
					}else{
199
						classification.addParentChild(nextHigherTaxon, genus, sourceRef, null);
200
					}
201
					nextHigherTaxon = genus;
202
					
203
					//subgenus
204
					String subGenusStr = CdmBase.deproxy(species.getName(), ZoologicalName.class).getInfraGenericEpithet();
205
					String subGenusAuthorStr = rs.getString("dtSpcSubgenaktauthor");
206
					boolean hasSubgenus = StringUtils.isNotBlank(subGenusStr) || StringUtils.isNotBlank(subGenusAuthorStr);
207
					if (hasSubgenus){
208
						Taxon subGenus = getTaxon(state, rs, genusStr, subGenusStr, Rank.SUBGENUS(), subGenusAuthorStr, taxonMap);
209
						classification.addParentChild(nextHigherTaxon, subGenus, sourceRef, null);
210
						nextHigherTaxon = subGenus;
211
					}
212
					
213
					classification.addParentChild(nextHigherTaxon, species, sourceRef, null);
214
					
215
					handleCountries(state, rs, species);
216
					
217
					handleCommonNames(state, rs, species);
218
					
219
					this.doIdCreatedUpdatedNotes(state, species, rs, taxonId, REFERENCE_NAMESPACE);
220
					
221
					objectsToSave.add(species); 
222
					
223

  
224
				} catch (Exception e) {
225
					logger.warn("Exception in current_species: IDcurrentspec " + taxonId + ". " + e.getMessage());
226
//					e.printStackTrace();
227
				} 
228
                
229
            }
230
           
231
//            logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
232

  
233
			logger.warn(pluralString + " to save: " + objectsToSave.size());
234
			getTaxonService().save(objectsToSave);	
235
			
236
			return success;
237
		} catch (SQLException e) {
238
			logger.error("SQLException:" +  e);
239
			return false;
240
		}
241
	}
242

  
243
	private void handleCountries(GlobisImportState state, ResultSet rs, Taxon species) throws SQLException {
244
		String countriesStr = rs.getString("dtSpcCountries");
245
		if (isBlank(countriesStr)){
246
			return;
247
		}
248
		String[] countriesSplit = countriesStr.split(";");
249
		for (String countryStr : countriesSplit){
250
			if (isBlank(countryStr)){
251
				continue;
252
			}else{
253
				countryStr = normalizeCountry(countryStr);
254
			}
255
			
256
			WaterbodyOrCountry country = WaterbodyOrCountry.getWaterbodyOrCountryByLabel(countryStr);
257
			if (country == null){
258
				try {
259
					country = (WaterbodyOrCountry)state.getTransformer().getNamedAreaByKey(countryStr);
260
				} catch (UndefinedTransformerMethodException e) {
261
					e.printStackTrace();
262
				}
263
			}
264
			
265
			if (country != null){
266
				TaxonDescription desc = getTaxonDescription(species, state.getTransactionalSourceReference(), false, true);
267
				Distribution distribution = Distribution.NewInstance(country, PresenceTerm.PRESENT());
268
				desc.addElement(distribution);
269
			}else{
270
				logger.warn("Country string not recognized: " + countryStr);
271
			}
272
		}
273
	}
274

  
275

  
276

  
277

  
278
	/**
279
	 * @param countryStr
280
	 * @return
281
	 */
282
	private String normalizeCountry(String countryStr) {
283
		String result = countryStr.trim();
284
		if (result.endsWith(".")){
285
			result = result.substring(0,result.length() - 1);
286
		}
287
		return result; 
288
	}
289
	
290
	private void handleCommonNames(GlobisImportState state, ResultSet rs, Taxon species) throws SQLException {
291
		String commonNamesStr = rs.getString("vernacularnames");
292
		if (isBlank(commonNamesStr)){
293
			return;
294
		}
295
		String[] commonNamesSplit = commonNamesStr.split(";");
296
		for (String commonNameStr : commonNamesSplit){
297
			if (isBlank(commonNameStr)){
298
				continue;
299
			}
300
			Language language = null; //TODO
301
			CommonTaxonName commonName = CommonTaxonName.NewInstance(commonNameStr, language);
302
			TaxonDescription desc = getTaxonDescription(species, state.getTransactionalSourceReference(), false, true);
303
			desc.addElement(commonName);
304
		}
305
	}
306

  
307

  
308

  
309

  
310
	/**
311
	 * Compares 2 taxa, returns true of both taxa look similar
312
	 * @param genus
313
	 * @param nextHigherTaxon
314
	 * @return
315
	 */
316
	private boolean compareTaxa(Taxon taxon1, Taxon taxon2) {
317
		ZoologicalName name1 = CdmBase.deproxy(taxon1.getName(), ZoologicalName.class);
318
		ZoologicalName name2 = CdmBase.deproxy(taxon2.getName(), ZoologicalName.class);
319
		if (!name1.getRank().equals(name2.getRank())){
320
			return false;
321
		}
322
		if (! name1.getTitleCache().equals(name2.getTitleCache())){
323
			return false;
324
		}
325
		return true;
326
	}
327

  
328

  
329

  
330

  
331
	private Taxon getParent(Taxon child, Classification classification) {
332
		for (TaxonNode node :  child.getTaxonNodes()){
333
			if (node.getClassification().equals(classification)){
334
				if (node.getParent() != null){
335
					return node.getParent().getTaxon();	
336
				}else{
337
					return null;
338
				}
339
			}
340
		}
341
		return null;
342
	}
343

  
344

  
345

  
346

  
347
	private Taxon getTaxon(GlobisImportState state, ResultSet rs, String uninomial, String infraGenericEpi, Rank rank, String author, Map<String, Taxon> taxonMap) {
348
		if (isBlank(uninomial)){
349
			return null;
350
		}
351
		
352
		String keyEpithet = StringUtils.isNotBlank(infraGenericEpi)? infraGenericEpi : uninomial ;
353
		
354
		String key = keyEpithet + "@" + CdmUtils.Nz(author) + "@" + rank.getTitleCache();
355
		Taxon taxon = taxonMap.get(key);
356
		if (taxon == null){
357
			ZoologicalName name = ZoologicalName.NewInstance(rank);
358
			name.setGenusOrUninomial(uninomial);
359
			if (isNotBlank(infraGenericEpi)){
360
				name.setInfraGenericEpithet(infraGenericEpi);
361
			}
362
			taxon = Taxon.NewInstance(name, state.getTransactionalSourceReference());
363
			
364
			taxonMap.put(key, taxon);
365
			handleAuthor(author, name);
366
			getTaxonService().save(taxon);
367
		}
368
		
369
		return taxon;
370
	}
371

  
372

  
373
	//fast and dirty is enough here
374
	private Classification classification;
375
	
376
	private Classification getClassification(GlobisImportState state) {
377
		if (this.classification == null){
378
			String name = state.getConfig().getClassificationName();
379
			Reference<?> reference = state.getTransactionalSourceReference();
380
			this.classification = Classification.NewInstance(name, reference, Language.DEFAULT());
381
			classification.setUuid(state.getConfig().getClassificationUuid());
382
			getClassificationService().save(classification);
383
		}
384
		return this.classification;
385
		
386
	}
387

  
388
	private INonViralNameParser parser = NonViralNameParserImpl.NewInstance();
389
	
390

  
391
	/* (non-Javadoc)
392
	 * @see eu.etaxonomy.cdm.io.common.mapping.IMappingImport#createObject(java.sql.ResultSet, eu.etaxonomy.cdm.io.common.ImportStateBase)
393
	 */
394
	public Taxon createObject(ResultSet rs, GlobisImportState state)
395
			throws SQLException {
396
		String speciesEpi = rs.getString("dtSpcSpcakt");
397
		String subGenusEpi = rs.getString("dtSpcSubgenakt");
398
		String genusEpi = rs.getString("dtSpcGenusakt");
399
		String author = rs.getString("dtSpcAutor");
400
		
401
		
402
		ZoologicalName zooName = ZoologicalName.NewInstance(Rank.SPECIES());
403
		zooName.setSpecificEpithet(speciesEpi);
404
		if (StringUtils.isNotBlank(subGenusEpi)){
405
			zooName.setInfraGenericEpithet(subGenusEpi);
406
		}
407
		zooName.setGenusOrUninomial(genusEpi);
408
		handleAuthor(author, zooName);
409
		
410
		Taxon taxon = Taxon.NewInstance(zooName, state.getTransactionalSourceReference());
411
		
412
		return taxon;
413
	}
414

  
415

  
416

  
417

  
418
	/**
419
	 * @param author
420
	 * @param zooName
421
	 */
422
	private void handleAuthor(String author, ZoologicalName zooName) {
423
		if (isBlank(author)){
424
			return;
425
		}
426
		try {
427
			if(author.matches(".*\\,\\s\\[\\d{4}\\].*")){
428
				author = author.replace("[", "").replace("]", "");
429
			}
430
			if (author.contains("?")){
431
				author = author.replace("H?bner", "H\u00fcbner");
432
				author = author.replace("Oberth?r", "Oberth\u00fcr");
433
			}
434
			
435
			parser.parseAuthors(zooName, author);
436
		} catch (StringNotParsableException e) {
437
			logger.warn("Author could not be parsed: " + author);
438
			zooName.setAuthorshipCache(author, true);
439
		}
440
	}
441

  
442
	/* (non-Javadoc)
443
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
444
	 */
445
	public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
446
		String nameSpace;
447
		Class cdmClass;
448
		Set<String> idSet;
449
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
450
		try{
451
			Set<String> taxonIdSet = new HashSet<String>();
452
			
453
			while (rs.next()){
454
//				handleForeignKey(rs, taxonIdSet, "taxonId");
455
			}
456
			
457
			//taxon map
458
			nameSpace = TAXON_NAMESPACE;
459
			cdmClass = Taxon.class;
460
			idSet = taxonIdSet;
461
			Map<String, Taxon> objectMap = (Map<String, Taxon>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
462
			result.put(nameSpace, objectMap);
463

  
464
			
465
		} catch (SQLException e) {
466
			throw new RuntimeException(e);
467
		}
468
		return result;	}
469
	
470
	/* (non-Javadoc)
471
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)
472
	 */
473
	@Override
474
	protected boolean doCheck(GlobisImportState state){
475
		IOValidator<GlobisImportState> validator = new GlobisCurrentSpeciesImportValidator();
476
		return validator.validate(state);
477
	}
478
	
479
	
480
	/* (non-Javadoc)
481
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
482
	 */
483
	protected boolean isIgnore(GlobisImportState state){
484
		return ! state.getConfig().isDoCurrentTaxa();
485
	}
486

  
487

  
488

  
489

  
490

  
491
}
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImageImport.java
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
4
* http://www.e-taxonomy.eu
5
* 
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

  
10
package eu.etaxonomy.cdm.io.globis;
11

  
12
import java.sql.ResultSet;
13
import java.sql.SQLException;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.Map;
17
import java.util.Set;
18

  
19
import org.apache.commons.lang.StringUtils;
20
import org.apache.log4j.Logger;
21
import org.springframework.stereotype.Component;
22

  
23
import eu.etaxonomy.cdm.io.common.IOValidator;
24
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
25
import eu.etaxonomy.cdm.io.globis.validation.GlobisCurrentSpeciesImportValidator;
26
import eu.etaxonomy.cdm.model.common.CdmBase;
27
import eu.etaxonomy.cdm.model.common.Language;
28
import eu.etaxonomy.cdm.model.media.Media;
29
import eu.etaxonomy.cdm.model.name.Rank;
30
import eu.etaxonomy.cdm.model.name.ZoologicalName;
31
import eu.etaxonomy.cdm.model.reference.Reference;
32
import eu.etaxonomy.cdm.model.taxon.Classification;
33
import eu.etaxonomy.cdm.model.taxon.Taxon;
34
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
35
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
36
import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
37
import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
38
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
39

  
40

  
41
/**
42
 * @author a.mueller
43
 * @created 20.02.2010
44
 * @version 1.0
45
 */
46
@Component
47
public class GlobisImageImport  extends GlobisImportBase<Taxon> {
48
	private static final Logger logger = Logger.getLogger(GlobisImageImport.class);
49
	
50
	private int modCount = 10000;
51
	private static final String pluralString = "images";
52
	private static final String dbTableName = "Einzelbilder";
53
	private static final Class cdmTargetClass = Media
54
	.class;  //not needed
55
	
56
	private static final String IMAGE_NAMESPACE = "Einzelbilder";
57
	
58
	public GlobisImageImport(){
59
		super(pluralString, dbTableName, cdmTargetClass);
60
	}
61

  
62

  
63
	
64
	
65
	/* (non-Javadoc)
66
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#getIdQuery()
67
	 */
68
	@Override
69
	protected String getIdQuery() {
70
		String strRecordQuery = 
71
			" SELECT BildId " + 
72
			" FROM " + dbTableName; 
73
		return strRecordQuery;	
74
	}
75

  
76

  
77

  
78

  
79
	/* (non-Javadoc)
80
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
81
	 */
82
	@Override
83
	protected String getRecordQuery(GlobisImportConfigurator config) {
84
		String strRecordQuery = 
85
			" SELECT i.*, NULL as Created_When, NULL as Created_Who," +
86
				"  NULL as Updated_who, NULL as Updated_When, NULL as Notes " + 
87
			" FROM " + getTableName() + " i " +
88
			" WHERE ( i.BildId IN (" + ID_LIST_TOKEN + ") )";
89
		return strRecordQuery;
90
	}
91
	
92

  
93

  
94
	/* (non-Javadoc)
95
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#doPartition(eu.etaxonomy.cdm.io.common.ResultSetPartitioner, eu.etaxonomy.cdm.io.globis.GlobisImportState)
96
	 */
97
	@Override
98
	public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) {
99
		boolean success = true;
100
		
101
		Set<TaxonBase> objectsToSave = new HashSet<TaxonBase>();
102
		
103
		Map<String, Taxon> taxonMap = (Map<String, Taxon>) partitioner.getObjectMap(TAXON_NAMESPACE);
104
//		Map<String, DerivedUnit> ecoFactDerivedUnitMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(ECO_FACT_DERIVED_UNIT_NAMESPACE);
105
		
106
		ResultSet rs = partitioner.getResultSet();
107

  
108
		Classification classification = getClassification(state);
109
		
110
		try {
111
			
112
			int i = 0;
113

  
114
			//for each reference
115
            while (rs.next()){
116
                
117
        		if ((i++ % modCount) == 0 && i!= 1 ){ logger.info(pluralString + " handled: " + (i-1));}
118
				
119
        		Integer taxonId = rs.getInt("IDcurrentspec");
120
        		
121
        		
122
        		//TODO
123
        		//String dtSpcJahr,
124
        		//dtSpcFamakt,dtSpcSubfamakt,dtSpcTribakt,
125
        		//fiSpcLiteratur, fiSpcspcgrptax, dtSpcCountries,vernacularnames
126
        		
127
				try {
128
					
129
					//source ref
130
					Reference<?> sourceRef = state.getTransactionalSourceReference();
131
					Taxon nextHigherTaxon = null;
132
					
133
					Taxon species = createObject(rs, state);
134
					
135
					//subgenus
136
					String subGenusStr = rs.getString("dtSpcSubgenakt");
137
					String subGenusAuthorStr = rs.getString("dtSpcSubgenaktauthor");
138
					boolean hasSubgenus = StringUtils.isNotBlank(subGenusStr) || StringUtils.isNotBlank(subGenusAuthorStr);
139
					if (hasSubgenus){
140
						Taxon subGenus = getTaxon(state, rs, subGenusStr, Rank.SUBGENUS(), subGenusAuthorStr, taxonMap);
141
						classification.addParentChild(subGenus, species, sourceRef, null);
142
						nextHigherTaxon = getParent(subGenus, classification);
143
					}
144
					
145
					//genus
146
					String genusStr = rs.getString("dtSpcGenusakt");
147
					String genusAuthorStr = rs.getString("dtSpcGenusaktauthor");
148
					Taxon genus = getTaxon(state, rs, genusStr, Rank.GENUS(), genusAuthorStr, taxonMap);
149
					if (nextHigherTaxon != null){
150
						if (! compareTaxa(genus, nextHigherTaxon)){
151
							logger.warn("Current genus and parent of subgenus are not equal: " + taxonId);
152
						}
153
					}else{
154
						classification.addParentChild(genus, species, sourceRef, null);
155
						nextHigherTaxon = getParent(genus, classification);
156
					}
157
					
158
					this.doIdCreatedUpdatedNotes(state, species, rs, taxonId, REFERENCE_NAMESPACE);
159
					
160
					objectsToSave.add(species); 
161
					
162

  
163
				} catch (Exception e) {
164
					logger.warn("Exception in current_species: IDcurrentspec " + taxonId + ". " + e.getMessage());
165
//					e.printStackTrace();
166
				} 
167
                
168
            }
169
           
170
//            logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
171

  
172
			logger.warn(pluralString + " to save: " + objectsToSave.size());
173
			getTaxonService().save(objectsToSave);	
174
			
175
			return success;
176
		} catch (SQLException e) {
177
			logger.error("SQLException:" +  e);
178
			return false;
179
		}
180
	}
181

  
182
	/**
183
	 * Compares 2 taxa, returns true of both taxa look similar
184
	 * @param genus
185
	 * @param nextHigherTaxon
186
	 * @return
187
	 */
188
	private boolean compareTaxa(Taxon taxon1, Taxon taxon2) {
189
		ZoologicalName name1 = CdmBase.deproxy(taxon1.getName(), ZoologicalName.class);
190
		ZoologicalName name2 = CdmBase.deproxy(taxon2.getName(), ZoologicalName.class);
191
		if (!name1.getRank().equals(name2.getRank())){
192
			return false;
193
		}
194
		if (! name1.getTitleCache().equals(name2.getTitleCache())){
195
			return false;
196
		}
197
		return true;
198
	}
199

  
200

  
201

  
202

  
203
	private Taxon getParent(Taxon subgenus, Classification classification) {
204
		for (TaxonNode node :  subgenus.getTaxonNodes()){
205
			if (node.getClassification().equals(classification)){
206
				return node.getParent().getTaxon();
207
			}
208
		}
209
		return null;
210
	}
211

  
212

  
213

  
214

  
215
	private Taxon getTaxon(GlobisImportState state, ResultSet rs, String subGenus, Rank rank, String author, Map<String, Taxon> taxonMap) {
216
		String key = subGenus + "@" + "subGenusAuthor" + "@" + rank.getTitleCache();
217
		Taxon taxon = taxonMap.get(key);
218
		if (taxon == null){
219
			ZoologicalName name = ZoologicalName.NewInstance(rank);
220
			taxon = Taxon.NewInstance(name, state.getTransactionalSourceReference());
221
			handleAuthor(author, name);
222
			getTaxonService().save(taxon);
223
		}
224
		
225
		return taxon;
226
	}
227

  
228

  
229
	//fast and dirty is enough here
230
	private Classification classification;
231
	
232
	private Classification getClassification(GlobisImportState state) {
233
		if (this.classification == null){
234
			String name = state.getConfig().getClassificationName();
235
			Reference<?> reference = state.getTransactionalSourceReference();
236
			this.classification = Classification.NewInstance(name, reference, Language.DEFAULT());
237
			classification.setUuid(state.getConfig().getClassificationUuid());
238
			getClassificationService().save(classification);
239
		}
240
		return this.classification;
241
		
242
	}
243

  
244
	private INonViralNameParser parser = NonViralNameParserImpl.NewInstance();
245
	
246

  
247
	/* (non-Javadoc)
248
	 * @see eu.etaxonomy.cdm.io.common.mapping.IMappingImport#createObject(java.sql.ResultSet, eu.etaxonomy.cdm.io.common.ImportStateBase)
249
	 */
250
	public Taxon createObject(ResultSet rs, GlobisImportState state)
251
			throws SQLException {
252
		String speciesEpi = rs.getString("dtSpcSpcakt");
253
		String subGenusEpi = rs.getString("dtSpcSubgenakt");
254
		String genusEpi = rs.getString("dtSpcGenusakt");
255
		String author = rs.getString("dtSpcAutor");
256
		
257
		
258
		ZoologicalName zooName = ZoologicalName.NewInstance(Rank.SPECIES());
259
		zooName.setSpecificEpithet(speciesEpi);
260
		if (StringUtils.isNotBlank(subGenusEpi)){
261
			zooName.setInfraSpecificEpithet(subGenusEpi);
262
		}
263
		zooName.setGenusOrUninomial(genusEpi);
264
		handleAuthor(author, zooName);
265
		
266
		Taxon taxon = Taxon.NewInstance(zooName, state.getTransactionalSourceReference());
267
		
268
		return taxon;
269
	}
270

  
271

  
272

  
273

  
274
	/**
275
	 * @param author
276
	 * @param zooName
277
	 */
278
	private void handleAuthor(String author, ZoologicalName zooName) {
279
		try {
280
			parser.parseAuthors(zooName, author);
281
		} catch (StringNotParsableException e) {
282
			logger.warn("Author could not be parsed: " + author);
283
			zooName.setAuthorshipCache(author, true);
284
		}
285
	}
286

  
287
	/* (non-Javadoc)
288
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
289
	 */
290
	public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
291
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
292
		return result;  //not needed
293
	}
294
	
295
	/* (non-Javadoc)
296
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)
297
	 */
298
	@Override
299
	protected boolean doCheck(GlobisImportState state){
300
		IOValidator<GlobisImportState> validator = new GlobisCurrentSpeciesImportValidator();
301
		return validator.validate(state);
302
	}
303
	
304
	
305
	/* (non-Javadoc)
306
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
307
	 */
308
	protected boolean isIgnore(GlobisImportState state){
309
		return ! state.getConfig().isDoImages();
310
	}
311

  
312

  
313

  
314

  
315

  
316
}
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportBase.java
20 20
import java.util.Set;
21 21
import java.util.UUID;
22 22

  
23
import org.apache.commons.lang.StringUtils;
23 24
import org.apache.log4j.Logger;
24 25
import org.joda.time.DateTime;
25 26

  
......
54 55
	
55 56
	//NAMESPACES
56 57
	
57
	protected static final String AREA_NAMESPACE = "gu";
58
	protected static final String DR_NAMESPACE = "dr";
59
	protected static final String IMAGE_NAMESPACE = "Images";
60
	protected static final String LINKS_NAMESPACE = "Links";
61
	protected static final String NOTES_NAMESPACE = "Notes";
62
	protected static final String LANGUAGE_NAMESPACE = "Language";
63
	protected static final String REFERENCE_NAMESPACE = "Source";
64
	protected static final String SOURCEUSE_NAMESPACE = "tu_sources";
65
	protected static final String TAXON_NAMESPACE = "Taxon";
66
	protected static final String NAME_NAMESPACE = "TaxonName";
67
	protected static final String VERNACULAR_NAMESPACE = "Vernaculars";
68
	protected static final String FEATURE_NAMESPACE = "note.type";
69
	protected static final String EXTENSION_TYPE_NAMESPACE = "ExtensionType";
70
	
71
	
58
	protected static final String REFERENCE_NAMESPACE = "Literatur";
59
	protected static final String TAXON_NAMESPACE = "current_species";
60

  
72 61

  
73 62
	private String pluralString;
74 63
	private String dbTableName;
......
109 98
		logger.info("end make " + getPluralString() + " ... " + getSuccessString(true));
110 99
		return;
111 100
	}
112
	
101

  
102

  
113 103
	public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) {
114 104
		boolean success = true ;
115 105
		Set objectsToSave = new HashSet();
......
137 127
	/**
138 128
	 * @return
139 129
	 */
140
	protected abstract DbImportMapping<?, ?> getMapping();
130
	protected /*abstract*/ DbImportMapping<?, ?> getMapping(){
131
		return null;
132
	}
141 133
	
142 134
	/**
143 135
	 * @return
......
222 214
		
223 215
		
224 216
		//notes
225
		if (CdmUtils.isNotEmpty(notes)){
217
		if (StringUtils.isNotBlank(notes)){
226 218
			String notesString = String.valueOf(notes);
227 219
			if (notesString.length() > 65530 ){
228 220
				notesString = notesString.substring(0, 65530) + "...";
......
237 229
	}
238 230
	
239 231
	private User getUser(String userString, GlobisImportState state){
240
		if (CdmUtils.isEmpty(userString)){
232
		if (StringUtils.isBlank(userString)){
241 233
			return null;
242 234
		}
243 235
		userString = userString.trim();
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportConfigurator.java
38 38
	}
39 39

  
40 40
	/* Max number of records to be saved with one service call */
41
	private int recordsPerTransaction = 1000;  //defaultValue
41
	private int recordsPerTransaction = 2000;  //defaultValue
42 42

  
43 43
	//TODO needed ??
44 44
	private Method userTransformationMethod;
45 45
	
46
	private boolean doVernaculars = true;
47
	private boolean doLinks = true;
48
	private boolean doNotes = true;
49 46
	private boolean doImages = true;
50
	private boolean doOccurrence = true;
47
	private boolean doCurrentTaxa = true;
48
	private boolean doSpecTaxa = true;
51 49
	private DO_REFERENCES doReferences = DO_REFERENCES.ALL;
52 50

  
53 51
	
......
58 56
				//ErmsGeneralImportValidator.class
59 57
				 GlobisReferenceImport.class
60 58
	//			, ErmsReferenceImport.class
61
//				, GlobisTaxonImport.class
59
				, GlobisCurrentSpeciesImport.class
60
				, GlobisSpecTaxImport.class
62 61
		};	
63 62
	}
64 63
	
......
163 162
	}
164 163

  
165 164

  
165
	public void setDoCurrentTaxa(boolean doCurrentTaxa) {
166
		this.doCurrentTaxa = doCurrentTaxa;
167
	}
168

  
169

  
170
	public boolean isDoCurrentTaxa() {
171
		return doCurrentTaxa;
172
	}
173

  
174

  
175
	public void setDoSpecTaxa(boolean doSpecTaxa) {
176
		this.doSpecTaxa = doSpecTaxa;
177
	}
178

  
179

  
180
	public boolean isDoSpecTaxa() {
181
		return doSpecTaxa;
182
	}
183

  
184

  
166 185
	
167 186

  
168 187
}
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisReferenceImport.java
12 12
import java.sql.ResultSet;
13 13
import java.sql.SQLException;
14 14
import java.util.HashMap;
15
import java.util.HashSet;
15 16
import java.util.Map;
17
import java.util.Set;
16 18

  
19
import org.apache.commons.lang.StringUtils;
17 20
import org.apache.log4j.Logger;
18 21
import org.springframework.stereotype.Component;
19 22

  
23
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
24
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
25
import eu.etaxonomy.cdm.io.algaterra.AlgaTerraImportState;
26
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonImport;
20 27
import eu.etaxonomy.cdm.io.common.IImportConfigurator;
21 28
import eu.etaxonomy.cdm.io.common.IOValidator;
29
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
22 30
import eu.etaxonomy.cdm.io.common.mapping.DbIgnoreMapper;
23 31
import eu.etaxonomy.cdm.io.common.mapping.DbImportExtensionMapper;
24 32
import eu.etaxonomy.cdm.io.common.mapping.DbImportMapping;
......
28 36
import eu.etaxonomy.cdm.io.common.mapping.IMappingImport;
29 37
import eu.etaxonomy.cdm.io.globis.validation.GlobisReferenceImportValidator;
30 38
import eu.etaxonomy.cdm.model.common.CdmBase;
39
import eu.etaxonomy.cdm.model.description.Feature;
40
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
41
import eu.etaxonomy.cdm.model.description.TaxonDescription;
42
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
31 43
import eu.etaxonomy.cdm.model.reference.Reference;
32 44
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
45
import eu.etaxonomy.cdm.model.reference.ReferenceType;
46
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
33 47

  
34 48

  
35 49
/**
......
41 55
public class GlobisReferenceImport  extends GlobisImportBase<Reference> implements IMappingImport<Reference, GlobisImportState>{
42 56
	private static final Logger logger = Logger.getLogger(GlobisReferenceImport.class);
43 57
	
44
	private DbImportMapping mapping;
45
	
46
	
47 58
	private int modCount = 10000;
48 59
	private static final String pluralString = "references";
49
	private static final String dbTableName = "literature";
60
	private static final String dbTableName = "Literatur";
50 61
	private static final Class cdmTargetClass = Reference.class;
51 62

  
52 63
	public GlobisReferenceImport(){
......
76 87
	@Override
77 88
	protected String getRecordQuery(GlobisImportConfigurator config) {
78 89
		String strRecordQuery = 
79
			" SELECT * " + 
80
			" FROM literature " +
81
			" WHERE ( literature.refId IN (" + ID_LIST_TOKEN + ") )";
90
			" SELECT l.*, l.DateCreated as Created_When, l.CreatedBy as Created_Who," +
91
			"        l.ModifiedBy as Updated_who, l.DateModified as Updated_When, l.RefRemarks as Notes " + 
92
			" FROM " + getTableName() + " l " +
93
			" WHERE ( l.refId IN (" + ID_LIST_TOKEN + ") )";
82 94
		return strRecordQuery;
83 95
	}
96
	
97

  
84 98

  
85 99
	/* (non-Javadoc)
86
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#getMapping()
100
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#doPartition(eu.etaxonomy.cdm.io.common.ResultSetPartitioner, eu.etaxonomy.cdm.io.globis.GlobisImportState)
87 101
	 */
88
	protected DbImportMapping getMapping() {
89
		if (mapping == null){
90
			mapping = new DbImportMapping();
91
			
92
			mapping.addMapper(DbImportObjectCreationMapper.NewInstance(this, "refID", REFERENCE_NAMESPACE)); //id
93
			mapping.addMapper(DbIgnoreMapper.NewInstance("CountryDummy"));
94
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("CreatedBy"));
95
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("DateCreated"));
96
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("DateModified"));
97
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("ModifiedBy"));
98
			mapping.addMapper(DbImportStringMapper.NewInstance("RefBookTitle", "title", false));
99
			//mapping.addMapper(DbImportTimePeriodMapper.NewInstance("RefDatePublished", "datePublished", false));
100
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("RefDatePublished"));
101
//			mapping.addMapper(DbImportExtensionTypeCreationMapper.NewInstance(dbIdAttribute, extensionTypeNamespace, dbTermAttribute, dbLabelAttribute, dbLabelAbbrevAttribute)
102
			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefEdition", GlobisTransformer.uuidEdition, "Edition", "Edition", "Ed."));
103
			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefEdition", GlobisTransformer.uuidEditor, "Editor", "Editor", "Editor"));
104
			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefGeneralKeywords", GlobisTransformer.uuidGeneralKeywords, "General Keywords", "General Keywords", "gen. keyw."));
105
			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefGeoKeywords", GlobisTransformer.uuidGeoKeywords, "Geographic Keywords", "Geo Keywords", "geo. keyw."));
106
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("RefIll only"));
107
			mapping.addMapper(DbImportStringMapper.NewInstance("ISSN", "issn", false));
108
			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefLibrary", GlobisTransformer.uuidLibrary, "Library", "Library", "Libr."));
109
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("RefMarker"));
110
			mapping.addMapper(DbImportStringMapper.NewInstance("RefPages", "pages"));
111
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("RefPages only"));
112
			
113
			
114
			
115
			Reference ref = null;
116
//			ref.setP
117
			
102
	@Override
103
	public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) {
104
		boolean success = true;
105
		
106
		Set<Reference> objectsToSave = new HashSet<Reference>();
107
		
108
//		Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
109
//		Map<String, DerivedUnit> ecoFactDerivedUnitMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(ECO_FACT_DERIVED_UNIT_NAMESPACE);
118 110
		
119
////			mapping.addMapper(DbImportExtensionMapper.NewInstance("imis_id", GlobisTransformer.IMIS_UUID, "imis", "imis", "imis"));
120
//						
121
//			mapping.addMapper(DbImportTruncatedStringMapper.NewInstance("source_name", "titleCache", "title"));
122
//			mapping.addMapper(DbImportStringMapper.NewInstance("source_abstract", "referenceAbstract"));
123
//			mapping.addMapper(DbImportAnnotationMapper.NewInstance("source_note", AnnotationType.EDITORIAL(), Language.DEFAULT()));
124
//			
125
//			//or as Extension?
126
//			mapping.addMapper(DbImportExtensionMapper.NewInstance("source_link", ExtensionType.URL()));
127
//			
128
//			//not yet implemented
129
//			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("source_type"));
130
//			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("source_orig_fn"));
111
		ResultSet rs = partitioner.getResultSet();
112

  
113
		try {
114
			
115
			int i = 0;
116

  
117
			//for each reference
118
            while (rs.next()){
119
                
120
        		if ((i++ % modCount) == 0 && i!= 1 ){ logger.info(pluralString + " handled: " + (i-1));}
121
				
122
        		Integer refId = rs.getInt("RefId");
123
        		String title = rs.getString("RefTitle");
124
        		String refJournal = rs.getString("RefJournal");
125
				
126
				try {
127
					
128
					//source ref
129
					Reference<?> sourceRef = state.getTransactionalSourceReference();
130
				
131
					Reference<?> ref = createObject(rs, state);
132
					ref.setTitle(title);
133
					
134
					if (StringUtils.isNotBlank(refJournal)){
135
						if (ref.getType().equals(ReferenceType.Article) ){
136
							Reference<?> journal = getJournal(state, rs, refJournal);
137
							ref.setInJournal(journal);
138
						}else{
139
							logger.warn("Reference type not supported for journal: " + ref.getType().toString());
140
						}
141
					}
142
					
143
					this.doIdCreatedUpdatedNotes(state, ref, rs, refId, REFERENCE_NAMESPACE);
144
					
145
					
146
					
147
					//DONE
148
//					RefType, RefTitle, RefJournal
149
					
150
					//TODO
151
					//Refid,CreatedBy,DateCreated,DateModified,ModifiedBy
152
					//RefAuthor,RefBookTitle,RefDatePublished
153
					//RefEdition, RefEditor, RefGeneralKeywords
154
					//RefGeoKeywords, RefIll only,RefISSN, RefJournal
155
					//RefLibrary, RefMarker,RefPages,RefPages only,
156
					//RefPlace, RefPublisher, RefRemarks,
157
					//RefSerial, RefSource, RefSpecificKeywords, RefTaxKeywords,
158
					//RefURL, RefVolPageFig, RefVolume, RefYear
159
					//SpecificKeywordDummy
160
					
161
					//no data
162
						//CountryDummy
163
					
164
					objectsToSave.add(ref); 
165
					
166

  
167
				} catch (Exception e) {
168
					logger.warn("Exception in literature: RefId " + refId + ". " + e.getMessage());
169
//					e.printStackTrace();
170
				} 
171
                
172
            }
173
           
174
//            logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
131 175

  
176
			logger.warn(pluralString + " to save: " + objectsToSave.size());
177
			getReferenceService().save(objectsToSave);	
178
			
179
			return success;
180
		} catch (SQLException e) {
181
			logger.error("SQLException:" +  e);
182
			return false;
132 183
		}
133
		return mapping;
134 184
	}
185

  
186

  
187

  
135 188
	
189
	private Reference<?> getJournal(GlobisImportState state, ResultSet rs, String refJournal) throws SQLException {
190
		
191
		
192
		Reference<?> journal = ReferenceFactory.newJournal();
193
		String issn = rs.getString("RefISSN");
194
		if (StringUtils.isNotBlank(issn)){
195
			issn.replaceAll("ISSN", "").trim();
196
			journal.setIssn(issn);			
197
		}
198

  
199
		
200
		
201
		//TODO deduplicate
202
		journal.setTitle(refJournal);
203
		return journal;
204
	}
205

  
206

  
207

  
208

  
136 209
	/* (non-Javadoc)
137 210
	 * @see eu.etaxonomy.cdm.io.common.mapping.IMappingImport#createObject(java.sql.ResultSet, eu.etaxonomy.cdm.io.common.ImportStateBase)
138 211
	 */
139
	public Reference createObject(ResultSet rs, GlobisImportState state)
212
	public Reference<?> createObject(ResultSet rs, GlobisImportState state)
140 213
			throws SQLException {
141
		Reference ref;
214
		Reference<?> ref;
142 215
		String refType = rs.getString("RefType");
143 216
		if (refType == null){
144 217
			ref = ReferenceFactory.newGeneric();
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisReferenceImport_Mapping_OLD.java
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
4
* http://www.e-taxonomy.eu
5
* 
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

  
10
package eu.etaxonomy.cdm.io.globis;
11

  
12
import java.sql.ResultSet;
13
import java.sql.SQLException;
14
import java.util.HashMap;
15
import java.util.Map;
16

  
17
import org.apache.log4j.Logger;
18
import org.springframework.stereotype.Component;
19

  
20
import eu.etaxonomy.cdm.io.common.IImportConfigurator;
21
import eu.etaxonomy.cdm.io.common.IOValidator;
22
import eu.etaxonomy.cdm.io.common.mapping.DbIgnoreMapper;
23
import eu.etaxonomy.cdm.io.common.mapping.DbImportExtensionMapper;
24
import eu.etaxonomy.cdm.io.common.mapping.DbImportMapping;
25
import eu.etaxonomy.cdm.io.common.mapping.DbImportObjectCreationMapper;
26
import eu.etaxonomy.cdm.io.common.mapping.DbImportStringMapper;
27
import eu.etaxonomy.cdm.io.common.mapping.DbNotYetImplementedMapper;
28
import eu.etaxonomy.cdm.io.common.mapping.IMappingImport;
29
import eu.etaxonomy.cdm.io.globis.validation.GlobisReferenceImportValidator;
30
import eu.etaxonomy.cdm.model.common.CdmBase;
31
import eu.etaxonomy.cdm.model.reference.Reference;
32
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
33

  
34

  
35
/**
36
 * @author a.mueller
37
 * @created 20.02.2010
38
 * @version 1.0
39
 */
40
@Component
41
public class GlobisReferenceImport_Mapping_OLD  extends GlobisImportBase<Reference> implements IMappingImport<Reference, GlobisImportState>{
42
	private static final Logger logger = Logger.getLogger(GlobisReferenceImport_Mapping_OLD.class);
43
	
44
	private DbImportMapping mapping;
45
	
46
	
47
	private int modCount = 10000;
48
	private static final String pluralString = "references";
49
	private static final String dbTableName = "Literatur";
50
	private static final Class cdmTargetClass = Reference.class;
51

  
52
	public GlobisReferenceImport_Mapping_OLD(){
53
		super(pluralString, dbTableName, cdmTargetClass);
54
	}
55

  
56

  
57
	
58
	
59
	/* (non-Javadoc)
60
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#getIdQuery()
61
	 */
62
	@Override
63
	protected String getIdQuery() {
64
		String strRecordQuery = 
65
			" SELECT refID " + 
66
			" FROM " + dbTableName; 
67
		return strRecordQuery;	
68
	}
69

  
70

  
71

  
72

  
73
	/* (non-Javadoc)
74
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
75
	 */
76
	@Override
77
	protected String getRecordQuery(GlobisImportConfigurator config) {
78
		String strRecordQuery = 
79
			" SELECT * " + 
80
			" FROM " + getTableName() + " l " +
81
			" WHERE ( l.refId IN (" + ID_LIST_TOKEN + ") )";
82
		return strRecordQuery;
83
	}
84

  
85
	/* (non-Javadoc)
86
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#getMapping()
87
	 */
88
	protected DbImportMapping getMapping() {
89
		if (mapping == null){
90
			mapping = new DbImportMapping();
91
			
92
			mapping.addMapper(DbImportObjectCreationMapper.NewInstance(this, "refID", REFERENCE_NAMESPACE)); //id
93
			mapping.addMapper(DbIgnoreMapper.NewInstance("CountryDummy"));
94
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("CreatedBy"));
95
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("DateCreated"));
96
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("DateModified"));
97
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("ModifiedBy"));
98
			mapping.addMapper(DbImportStringMapper.NewInstance("RefBookTitle", "title", false));
99
			//mapping.addMapper(DbImportTimePeriodMapper.NewInstance("RefDatePublished", "datePublished", false));
100
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("RefDatePublished"));
101
//			mapping.addMapper(DbImportExtensionTypeCreationMapper.NewInstance(dbIdAttribute, extensionTypeNamespace, dbTermAttribute, dbLabelAttribute, dbLabelAbbrevAttribute)
102
//			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefEdition", GlobisTransformer.uuidEdition, "Edition", "Edition", "Ed."));
103
//			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefEdition", GlobisTransformer.uuidEditor, "Editor", "Editor", "Editor"));
104
//			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefGeneralKeywords", GlobisTransformer.uuidGeneralKeywords, "General Keywords", "General Keywords", "gen. keyw."));
105
//			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefGeoKeywords", GlobisTransformer.uuidGeoKeywords, "Geographic Keywords", "Geo Keywords", "geo. keyw."));
106
//			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("RefIll only"));
107
//			mapping.addMapper(DbImportStringMapper.NewInstance("ISSN", "issn", false));
108
//			mapping.addMapper(DbImportExtensionMapper.NewInstance("RefLibrary", GlobisTransformer.uuidLibrary, "Library", "Library", "Libr."));
109
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("RefMarker"));
110
			mapping.addMapper(DbImportStringMapper.NewInstance("RefPages", "pages"));
111
			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("RefPages only"));
112
			
113
			
114
			
115
			Reference ref = null;
116
//			ref.setP
117
			
118
		
119
////			mapping.addMapper(DbImportExtensionMapper.NewInstance("imis_id", GlobisTransformer.IMIS_UUID, "imis", "imis", "imis"));
120
//						
121
//			mapping.addMapper(DbImportTruncatedStringMapper.NewInstance("source_name", "titleCache", "title"));
122
//			mapping.addMapper(DbImportStringMapper.NewInstance("source_abstract", "referenceAbstract"));
123
//			mapping.addMapper(DbImportAnnotationMapper.NewInstance("source_note", AnnotationType.EDITORIAL(), Language.DEFAULT()));
124
//			
125
//			//or as Extension?
126
//			mapping.addMapper(DbImportExtensionMapper.NewInstance("source_link", ExtensionType.URL()));
127
//			
128
//			//not yet implemented
129
//			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("source_type"));
130
//			mapping.addMapper(DbNotYetImplementedMapper.NewInstance("source_orig_fn"));
131

  
132
		}
133
		return mapping;
134
	}
135
	
136
	/* (non-Javadoc)
137
	 * @see eu.etaxonomy.cdm.io.common.mapping.IMappingImport#createObject(java.sql.ResultSet, eu.etaxonomy.cdm.io.common.ImportStateBase)
138
	 */
139
	public Reference createObject(ResultSet rs, GlobisImportState state)
140
			throws SQLException {
141
		Reference ref;
142
		String refType = rs.getString("RefType");
143
		if (refType == null){
144
			ref = ReferenceFactory.newGeneric();
145
		}else if (refType == "book"){
146
			ref = ReferenceFactory.newBook();
147
		}else if (refType == "paper in journal"){
148
			ref = ReferenceFactory.newArticle();
149
		}else if (refType.startsWith("unpublished") ){
150
			ref = ReferenceFactory.newGeneric();
151
		}else if (refType.endsWith("paper in journal")){
152
			ref = ReferenceFactory.newArticle();
153
		}else if (refType == "paper in book"){
154
			ref = ReferenceFactory.newBookSection();
155
		}else if (refType == "paper in journalwebsite"){
156
			ref = ReferenceFactory.newArticle();
157
		}else{
158
			logger.warn("Unknown reference type: " + refType);
159
			ref = ReferenceFactory.newGeneric();
160
		}
161
		return ref;
162
	}
163

  
164
	/* (non-Javadoc)
165
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
166
	 */
167
	public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
168
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
169
		return result;  //not needed
170
	}
171
	
172
	/* (non-Javadoc)
173
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)
174
	 */
175
	@Override
176
	protected boolean doCheck(GlobisImportState state){
177
		IOValidator<GlobisImportState> validator = new GlobisReferenceImportValidator();
178
		return validator.validate(state);
179
	}
180
	
181
	
182
	/* (non-Javadoc)
183
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
184
	 */
185
	protected boolean isIgnore(GlobisImportState state){
186
		//TODO
187
		return state.getConfig().getDoReferences() != IImportConfigurator.DO_REFERENCES.ALL;
188
	}
189

  
190

  
191

  
192

  
193

  
194
}
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisSpecTaxImport.java
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
4
* http://www.e-taxonomy.eu
5
* 
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

  
10
package eu.etaxonomy.cdm.io.globis;
11

  
12
import java.sql.ResultSet;
13
import java.sql.SQLException;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.Map;
17
import java.util.Set;
18

  
19
import org.apache.commons.lang.StringUtils;
20
import org.apache.log4j.Logger;
21
import org.springframework.stereotype.Component;
22

  
23
import eu.etaxonomy.cdm.io.common.IImportConfigurator;
24
import eu.etaxonomy.cdm.io.common.IOValidator;
25
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
26
import eu.etaxonomy.cdm.io.common.mapping.IMappingImport;
27
import eu.etaxonomy.cdm.io.globis.validation.GlobisReferenceImportValidator;
28
import eu.etaxonomy.cdm.io.globis.validation.GlobisSpecTaxaImportValidator;
29
import eu.etaxonomy.cdm.model.common.CdmBase;
30
import eu.etaxonomy.cdm.model.reference.Reference;
31
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
32
import eu.etaxonomy.cdm.model.reference.ReferenceType;
33

  
34

  
35
/**
36
 * @author a.mueller
37
 * @created 20.02.2010
38
 * @version 1.0
39
 */
40
@Component
41
public class GlobisSpecTaxImport  extends GlobisImportBase<Reference> implements IMappingImport<Reference, GlobisImportState>{
42
	private static final Logger logger = Logger.getLogger(GlobisSpecTaxImport.class);
43
	
44
	private int modCount = 10000;
45
	private static final String pluralString = "taxa";
46
	private static final String dbTableName = "specTax";
47
	private static final Class cdmTargetClass = Reference.class;
48

  
49
	public GlobisSpecTaxImport(){
50
		super(pluralString, dbTableName, cdmTargetClass);
51
	}
52

  
53

  
54
	
55
	
56
	/* (non-Javadoc)
57
	 * @see eu.etaxonomy.cdm.io.globis.GlobisImportBase#getIdQuery()
58
	 */
59
	@Override
60
	protected String getIdQuery() {
61
		String strRecordQuery = 
62
			" SELECT specTaxId " + 
63
			" FROM " + dbTableName; 
64
		return strRecordQuery;	
65
	}
66

  
67

  
68

  
69

  
70
	/* (non-Javadoc)
71
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
72
	 */
73
	@Override
74
	protected String getRecordQuery(GlobisImportConfigurator config) {
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff