Project

General

Profile

« Previous | Next » 

Revision 495cebda

Added by Andreas Müller about 11 years ago

updates to globis import

View differences:

app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisCommonNameImport.java
16 16
import java.util.Map;
17 17
import java.util.Set;
18 18

  
19
import org.apache.commons.lang.StringUtils;
20 19
import org.apache.log4j.Logger;
21 20
import org.springframework.stereotype.Component;
22 21

  
23
import eu.etaxonomy.cdm.common.CdmUtils;
24
import eu.etaxonomy.cdm.io.common.IOValidator;
25 22
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
26
import eu.etaxonomy.cdm.io.globis.validation.GlobisCurrentSpeciesImportValidator;
23
import eu.etaxonomy.cdm.model.agent.Person;
24
import eu.etaxonomy.cdm.model.agent.Team;
25
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
27 26
import eu.etaxonomy.cdm.model.common.CdmBase;
28 27
import eu.etaxonomy.cdm.model.common.Language;
29
import eu.etaxonomy.cdm.model.description.Distribution;
30
import eu.etaxonomy.cdm.model.description.PresenceTerm;
28
import eu.etaxonomy.cdm.model.common.TimePeriod;
29
import eu.etaxonomy.cdm.model.description.CommonTaxonName;
31 30
import eu.etaxonomy.cdm.model.description.TaxonDescription;
31
import eu.etaxonomy.cdm.model.location.NamedArea;
32 32
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
33
import eu.etaxonomy.cdm.model.name.Rank;
34
import eu.etaxonomy.cdm.model.name.ZoologicalName;
35 33
import eu.etaxonomy.cdm.model.reference.Reference;
36
import eu.etaxonomy.cdm.model.taxon.Classification;
34
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
37 35
import eu.etaxonomy.cdm.model.taxon.Taxon;
38 36
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
39
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
40 37

  
41 38

  
42 39
/**
......
57 54
		super(pluralString, dbTableName, cdmTargetClass);
58 55
	}
59 56

  
60

  
57
	//dirty but acceptable for globis environment
58
	private Map<Integer,Reference> refMap = new HashMap<Integer,Reference>();
61 59
	
62 60
	
63 61
	/* (non-Javadoc)
......
111 109
                
112 110
        		if ((i++ % modCount) == 0 && i!= 1 ){ logger.info(pluralString + " handled: " + (i-1));}
113 111
				
114
        		Integer taxonId = rs.getInt("IdCrrentSpec");
115
        		
116
        		
117
				try {
112
        		Integer idTaxon = nullSafeInt(rs,"IDCurrentSpec");
113
				
114
        		try {
118 115
					
119 116
					//source ref
120 117
					Reference<?> sourceRef = state.getTransactionalSourceReference();
121 118
			
122
					//species
123
					Taxon species = createObject(rs, state);
124
					
119
					//common names
120
					Integer id = nullSafeInt(rs,"ID");
121
					String isoLang = rs.getString("ISO");
122
					String strCommonName = rs.getString("commonname");
123
					Integer refID = nullSafeInt(rs,"ReferenceID");
124
					String strCountryCode = rs.getString("Code2");
125 125
					
126
					handleCountries(state, rs, species);
127
					
128
					this.doIdCreatedUpdatedNotes(state, species, rs, taxonId, TAXON_NAMESPACE);
129
					
130
					objectsToSave.add(species); 
131 126
					
127
					Taxon taxon = taxonMap.get(String.valueOf(idTaxon));
128
					if (taxon == null){
129
						logger.warn("No taxon found for taxonId " + idTaxon);
130
					}else if (isBlank(strCommonName)){
131
						logger.warn("No common name string defined for common name ID: " + id);
132
					}else{
133
						Language language = getLanguage(isoLang);
134
						if (language == null){
135
							logger.warn("No language found for common name ID: " + id);
136
						}
137
						NamedArea area = WaterbodyOrCountry.getWaterbodyOrCountryByIso3166A2(strCountryCode);
138
						if (area == null){
139
							logger.warn("No country found for common name ID: " + id);
140
						}
141

  
142
						TaxonDescription taxonDescription = getTaxonDescription(taxon, sourceRef, ! IMAGE_GALLERY,  CREATE);
143
						CommonTaxonName commonName = CommonTaxonName.NewInstance(strCommonName, language, area);
144
						taxonDescription.addElement(commonName);
145
						
146
						Reference<?> ref = handleReference(state, refID);
147
						if (ref == null && refID != null){
148
							logger.warn("No reference found for common name ID: " + id);
149
						}else{
150
							commonName.addSource(String.valueOf(refID), "reference", sourceRef, null);
151
						}
152
						
153
						objectsToSave.add(taxon); 
154
					}
132 155

  
133 156
				} catch (Exception e) {
134
					logger.warn("Exception in current_species: IDcurrentspec " + taxonId + ". " + e.getMessage());
135
//					e.printStackTrace();
157
					logger.warn("Exception in current_species: IDcurrentspec " + idTaxon + ". " + e.getMessage());
158
					e.printStackTrace();
136 159
				} 
137 160
                
138 161
            }
......
149 172
		}
150 173
	}
151 174

  
152
	private void handleCountries(GlobisImportState state, ResultSet rs, Taxon species) throws SQLException {
153
		String countriesStr = rs.getString("dtSpcCountries");
154
		if (isBlank(countriesStr)){
155
			return;
156
		}
157
		String[] countriesSplit = countriesStr.split(";");
158
		for (String countryStr : countriesSplit){
159
			if (isBlank(countryStr)){
160
				continue;
161
			}
162
			countryStr = countryStr.trim();
163
			
164
			//TODO use isComplete
165
			boolean isComplete = countryStr.endsWith(".");
166
			if (isComplete){
167
				countryStr = countryStr.substring(0,countryStr.length() - 1).trim();
168
			}
169
			boolean isDoubtful = countryStr.endsWith("[?]");
170
			if (isDoubtful){
171
				countryStr = countryStr.substring(0,countryStr.length() - 3).trim();
172
			}
173
			if (countryStr.startsWith("?")){
174
				isDoubtful = true;
175
				countryStr = countryStr.substring(1).trim();
176
			}
177
			
178
			
179
			
180
			countryStr = normalizeCountry(countryStr);
181
			
182
			WaterbodyOrCountry country = getCountry(state, countryStr);
183
			
184
			PresenceTerm status;
185
			if (isDoubtful){
186
				status = PresenceTerm.PRESENT_DOUBTFULLY();
187
			}else{
188
				status = PresenceTerm.PRESENT();
189
			}
190
			
191
			if (country != null){
192
				TaxonDescription desc = getTaxonDescription(species, state.getTransactionalSourceReference(), false, true);
193
				Distribution distribution = Distribution.NewInstance(country, status);
194
				desc.addElement(distribution);
195
			}else{
196
				logger.warn("Country string not recognized: " + countryStr);
175
	
176

  
177
	private Map<String,Language> languageMap = new HashMap<String,Language>();
178
	private Language getLanguage(String isoLang) {
179
		Language result = languageMap.get(isoLang);
180
		if (result == null){
181
		
182
			result = getTermService().getLanguageByIso(isoLang);
183
			if (result == null){
184
				logger.warn("No language found for iso code: " + isoLang);
197 185
			}
198 186
		}
199
	}
200

  
201

  
187
		return result;
202 188

  
203
	/**
204
	 * @param countryStr
205
	 * @return
206
	 */
207
	private String normalizeCountry(String countryStr) {
208
		String result = countryStr.trim();
209
		if (result.endsWith(".")){
210
			result = result.substring(0,result.length() - 1);
211
		}
212
		return result; 
213 189
	}
214 190
	
215

  
216
	/* (non-Javadoc)
217
	 * @see eu.etaxonomy.cdm.io.common.mapping.IMappingImport#createObject(java.sql.ResultSet, eu.etaxonomy.cdm.io.common.ImportStateBase)
218
	 */
219
	public Taxon createObject(ResultSet rs, GlobisImportState state)
220
			throws SQLException {
221
		String speciesEpi = rs.getString("dtSpcSpcakt");
222
		String subGenusEpi = rs.getString("dtSpcSubgenakt");
223
		String genusEpi = rs.getString("dtSpcGenusakt");
224
		String author = rs.getString("dtSpcAutor");
225
		
191
	private Reference<?> handleReference(GlobisImportState state, Integer refId){
192
		Reference<?> result = refMap.get(refId);
226 193
		
227
		ZoologicalName zooName = ZoologicalName.NewInstance(Rank.SPECIES());
228
		zooName.setSpecificEpithet(speciesEpi);
229
		if (StringUtils.isNotBlank(subGenusEpi)){
230
			zooName.setInfraGenericEpithet(subGenusEpi);
194
		if (result == null){
195
			try {
196
				String sql = "SELECT * FROM references WHERE ReferenceID = " + refId;
197
				ResultSet rs = state.getConfig().getSource().getResultSet(sql);
198
				rs.next();
199
				
200
				String authors = rs.getString("Author(s)");
201
				String title = rs.getString("Title");
202
				String details = rs.getString("Details");
203
				Integer year = nullSafeInt(rs, "year");
204
				result = ReferenceFactory.newGeneric();
205
				result.setTitleCache(details);
206
				result.setTitle(title);
207
				result.setDatePublished(TimePeriod.NewInstance(year));
208

  
209
				TeamOrPersonBase<?> author;
210
				String[] authorSplit = authors.split("&");
211
				if (authorSplit.length > 1){
212
					Team team = Team.NewInstance();
213
					author = team;
214
					for (String singleAuthor : authorSplit){
215
						Person person = makeSingleAuthor(singleAuthor);
216
						team.addTeamMember(person);
217
					}
218
				}else{
219
					author = makeSingleAuthor(authors);
220
				}
221
				
222
				result.setAuthorTeam(author);
223
				refMap.put(refId,result);
224
				rs.close();
225
			} catch (SQLException e) {
226
				e.printStackTrace();
227
			}
228

  
229
			
231 230
		}
232
		zooName.setGenusOrUninomial(genusEpi);
233
		handleAuthorAndYear(author, zooName);
234 231
		
235
		Taxon taxon = Taxon.NewInstance(zooName, state.getTransactionalSourceReference());
236
		
237
		return taxon;
232
		return result;
238 233
	}
239 234

  
240 235

  
241 236

  
242 237

  
238
	private Person makeSingleAuthor(String authors) {
239
		Person result = Person.NewTitledInstance(authors);
240
		return result;
241
	}
242

  
243

  
244

  
243 245

  
244 246
	/* (non-Javadoc)
245 247
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
......
253 255
			Set<String> taxonIdSet = new HashSet<String>();
254 256
			
255 257
			while (rs.next()){
256
//				handleForeignKey(rs, taxonIdSet, "taxonId");
258
				handleForeignKey(rs, taxonIdSet, "IDCurrentSpec");
257 259
			}
258 260
			
259 261
			//taxon map
......
275 277
	 */
276 278
	@Override
277 279
	protected boolean doCheck(GlobisImportState state){
278
		IOValidator<GlobisImportState> validator = new GlobisCurrentSpeciesImportValidator();
279
		return validator.validate(state);
280
//		IOValidator<GlobisImportState> validator = new GlobisCurrentSpeciesImportValidator();
281
		return true;
280 282
	}
281 283
	
282 284
	
......
284 286
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
285 287
	 */
286 288
	protected boolean isIgnore(GlobisImportState state){
287
		return ! state.getConfig().isDoCurrentTaxa();
289
		return ! state.getConfig().isDoCommonNames();
288 290
	}
289 291

  
290 292

  
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisCurrentSpeciesImport.java
29 29
import eu.etaxonomy.cdm.model.description.Distribution;
30 30
import eu.etaxonomy.cdm.model.description.PresenceTerm;
31 31
import eu.etaxonomy.cdm.model.description.TaxonDescription;
32
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
32
import eu.etaxonomy.cdm.model.location.NamedArea;
33 33
import eu.etaxonomy.cdm.model.name.Rank;
34 34
import eu.etaxonomy.cdm.model.name.ZoologicalName;
35 35
import eu.etaxonomy.cdm.model.reference.Reference;
......
116 116
				
117 117
        		Integer taxonId = rs.getInt("IDcurrentspec");
118 118
        		
119
        		
120 119
        		//String dtSpcJahr -> ignore !
121 120
        		//empty: fiSpcLiteratur
122 121
        		
......
134 133
					boolean hasNewParent = false; //true if any parent is new
135 134
					
136 135
					//species
137
					Taxon species = createObject(rs, state);
136
					Taxon species = createObject(rs, state, taxonId);
138 137
					
139 138
					
140 139
					String familyStr = rs.getString("dtSpcFamakt");
......
142 141
					String tribeStr = rs.getString("dtSpcTribakt");
143 142
					
144 143
					//family
145
					Taxon family = getTaxon(state, rs, familyStr, null, Rank.FAMILY(), null, taxonMap);
144
					Taxon family = getTaxon(state, rs, familyStr, null, Rank.FAMILY(), null, taxonMap, taxonId);
146 145
					
147 146
					//subfamily
148
					Taxon subFamily = getTaxon(state, rs, subFamilyStr, null, Rank.SUBFAMILY(), null, taxonMap);
147
					Taxon subFamily = getTaxon(state, rs, subFamilyStr, null, Rank.SUBFAMILY(), null, taxonMap, taxonId);
149 148
					Taxon subFamilyParent = getParent(subFamily, classification);
150 149
					if (subFamilyParent != null){
151 150
						if (! compareTaxa(family, subFamilyParent)){
......
157 156
					nextHigherTaxon = subFamily;
158 157
					
159 158
					//tribe
160
					Taxon tribe = getTaxon(state, rs, tribeStr, null, Rank.TRIBE(), null, taxonMap);
159
					Taxon tribe = getTaxon(state, rs, tribeStr, null, Rank.TRIBE(), null, taxonMap, taxonId);
161 160
					if (tribe != null){
162 161
						Taxon tribeParent = getParent(tribe, classification);
163 162
						if (tribeParent != null){
......
174 173
					//genus
175 174
					String genusStr = rs.getString("dtSpcGenusakt");
176 175
					String genusAuthorStr = rs.getString("dtSpcGenusaktauthor");
177
					Taxon genus = getTaxon(state, rs, genusStr, null, Rank.GENUS(), genusAuthorStr, taxonMap);
176
					Taxon genus = getTaxon(state, rs, genusStr, null, Rank.GENUS(), genusAuthorStr, taxonMap, taxonId);
178 177
					Taxon genusParent = getParent(genus, classification);
179 178
					
180 179
					if (genusParent != null){
......
191 190
					String subGenusAuthorStr = rs.getString("dtSpcSubgenaktauthor");
192 191
					boolean hasSubgenus = StringUtils.isNotBlank(subGenusStr) || StringUtils.isNotBlank(subGenusAuthorStr);
193 192
					if (hasSubgenus){
194
						Taxon subGenus = getTaxon(state, rs, genusStr, subGenusStr, Rank.SUBGENUS(), subGenusAuthorStr, taxonMap);
193
						Taxon subGenus = getTaxon(state, rs, genusStr, subGenusStr, Rank.SUBGENUS(), subGenusAuthorStr, taxonMap, taxonId);
195 194
						classification.addParentChild(nextHigherTaxon, subGenus, sourceRef, null);
196 195
						nextHigherTaxon = subGenus;
197 196
					}
......
256 255
			
257 256
			countryStr = normalizeCountry(countryStr);
258 257
			
259
			WaterbodyOrCountry country = getCountry(state, countryStr);
258
			NamedArea country = getCountry(state, countryStr);
260 259
			
261 260
			PresenceTerm status;
262 261
			if (isDoubtful){
......
348 347

  
349 348

  
350 349

  
351
	private Taxon getTaxon(GlobisImportState state, ResultSet rs, String uninomial, String infraGenericEpi, Rank rank, String author, Map<String, Taxon> taxonMap) {
350
	private Taxon getTaxon(GlobisImportState state, ResultSet rs, String uninomial, String infraGenericEpi, Rank rank, String author, Map<String, Taxon> taxonMap, Integer taxonId) {
352 351
		if (isBlank(uninomial)){
353 352
			return null;
354 353
		}
......
366 365
			taxon = Taxon.NewInstance(name, state.getTransactionalSourceReference());
367 366
			
368 367
			taxonMap.put(key, taxon);
369
			handleAuthorAndYear(author, name);
368
			handleAuthorAndYear(author, name, taxonId);
370 369
			getTaxonService().save(taxon);
371 370
		}
372 371
		
......
392 391
	/* (non-Javadoc)
393 392
	 * @see eu.etaxonomy.cdm.io.common.mapping.IMappingImport#createObject(java.sql.ResultSet, eu.etaxonomy.cdm.io.common.ImportStateBase)
394 393
	 */
395
	public Taxon createObject(ResultSet rs, GlobisImportState state)
394
	public Taxon createObject(ResultSet rs, GlobisImportState state, Integer taxonId)
396 395
			throws SQLException {
397 396
		String speciesEpi = rs.getString("dtSpcSpcakt");
398 397
		String subGenusEpi = rs.getString("dtSpcSubgenakt");
......
406 405
			zooName.setInfraGenericEpithet(subGenusEpi);
407 406
		}
408 407
		zooName.setGenusOrUninomial(genusEpi);
409
		handleAuthorAndYear(author, zooName);
408
		handleAuthorAndYear(author, zooName, taxonId);
410 409
		
411 410
		Taxon taxon = Taxon.NewInstance(zooName, state.getTransactionalSourceReference());
412 411
		
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImageImport.java
36 36
import eu.etaxonomy.cdm.model.common.Language;
37 37
import eu.etaxonomy.cdm.model.common.Marker;
38 38
import eu.etaxonomy.cdm.model.common.MarkerType;
39
import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
40
import eu.etaxonomy.cdm.model.description.Feature;
41
import eu.etaxonomy.cdm.model.description.TaxonDescription;
42
import eu.etaxonomy.cdm.model.description.TextData;
43 39
import eu.etaxonomy.cdm.model.media.Media;
44 40
import eu.etaxonomy.cdm.model.name.ZoologicalName;
45 41
import eu.etaxonomy.cdm.model.occurrence.Collection;
......
48 44
import eu.etaxonomy.cdm.model.reference.Reference;
49 45
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
50 46
import eu.etaxonomy.cdm.model.taxon.Taxon;
51
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
52 47

  
53 48

  
54 49
/**
......
60 55
public class GlobisImageImport  extends GlobisImportBase<Taxon> {
61 56
	private static final Logger logger = Logger.getLogger(GlobisImageImport.class);
62 57
	
63
	private int modCount = 10000;
58
	private int modCount = 1000;
64 59

  
65 60
	private UUID uuidArtNonSpecTaxMarkerType = UUID.fromString("be362085-0f5b-4314-96d1-78b9b129ef6d") ;
66 61
	private static final String pluralString = "images";
67 62
	private static final String dbTableName = "Einzelbilder";
68 63
	private static final Class cdmTargetClass = Media.class;  //not needed
69 64
	
70
	private static final String IMAGE_NAMESPACE = "Einzelbilder";
71 65
	private static UUID uuidGartRef = UUID.fromString("af85470f-6e54-4304-9d29-fd117cd56161"); 
72 66
	
73 67
	public GlobisImageImport(){
......
116 110
		
117 111
		Set<Media> objectsToSave = new HashSet<Media>();
118 112
		
119
		Map<String, Specimen> typeMap = (Map<String, Specimen>) partitioner.getObjectMap(GlobisSpecTaxImport.TYPE_NAMESPACE);
113
		Map<String, Specimen> typeMap = (Map<String, Specimen>) partitioner.getObjectMap(TYPE_NAMESPACE);
120 114
		
121 115
		Map<String, Taxon> taxonMap = (Map<String, Taxon>) partitioner.getObjectMap(TAXON_NAMESPACE);
122
		Map<String, ZoologicalName> specTaxNameMap = (Map<String, ZoologicalName>) partitioner.getObjectMap(GlobisSpecTaxImport.SPEC_TAX_NAMESPACE);
116
		Map<String, ZoologicalName> specTaxNameMap = (Map<String, ZoologicalName>) partitioner.getObjectMap(SPEC_TAX_NAMESPACE);
123 117
		
124 118
		ResultSet rs = partitioner.getResultSet();
125 119
		
......
192 186
					
193 187
					//GART id (specimenID)
194 188
					if (isNotBlank(specimenId)){
195
						specimen.addSource(specimenId, "", refGart, null);
189
						specimen.addSource(specimenId, "specimenId", refGart, null);
196 190
					}
197 191
					//bemerkungen
198 192
					if (isNotBlank(bemerkungen)){
......
294 288

  
295 289
	private String getNameFromFileOs(ResultSet rs) throws SQLException {
296 290
		String fileOS = rs.getString("file OS");
297
		Pattern pattern = Pattern.compile("(.+)(_...._..\\.jpg)");
291
		Pattern pattern = Pattern.compile("(.+)(_.{4}(-.{1,3})?(_Nr\\d{3,4})?_.{2,3}\\.jpg)");
298 292
		Matcher matcher = pattern.matcher(fileOS);
299 293
		if (matcher.matches()){
300 294
			String match = matcher.group(1);
......
433 427
			
434 428
			while (rs.next()){
435 429
				handleForeignKey(rs, currSpecIdSet, "SpecCurrspecID");
430
				handleForeignKey(rs, specTaxIdSet, "spectaxID");
436 431
				handleTypeKey(rs, typeIdSet, "spectaxID", "copyright");
437 432
			}
438 433
			
439 434
			//specTax map
440
			nameSpace = GlobisSpecTaxImport.SPEC_TAX_NAMESPACE;
441
			cdmClass = TaxonBase.class;
435
			nameSpace = SPEC_TAX_NAMESPACE;
436
			cdmClass = ZoologicalName.class;
442 437
			idSet = specTaxIdSet;
443
			Map<String, TaxonBase> specTaxMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
444
			result.put(nameSpace, specTaxMap);
438
			Map<String, ZoologicalName> specTaxNameMap = (Map<String, ZoologicalName>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
439
			result.put(nameSpace, specTaxNameMap);
445 440

  
446
			//taxon map
447
			nameSpace = TAXON_NAMESPACE;
448
			cdmClass = Taxon.class;
449
			idSet = currSpecIdSet;
450
			Map<String, Taxon> taxonMap = (Map<String, Taxon>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
451
			result.put(nameSpace, taxonMap);
441
//			//taxon map
442
//			nameSpace = TAXON_NAMESPACE;
443
//			cdmClass = Taxon.class;
444
//			idSet = currSpecIdSet;
445
//			Map<String, Taxon> taxonMap = (Map<String, Taxon>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
446
//			result.put(nameSpace, taxonMap);
452 447

  
453 448
			
454 449
			//type map
......
465 460
		return result;
466 461
	}
467 462
	
468
	private void handleTypeKey(ResultSet rs, Set<String> idSet, String specTaxIdAttr, String copyrightAttr)
469
			throws SQLException {
463
	private void handleTypeKey(ResultSet rs, Set<String> idSet, String specTaxIdAttr, String copyrightAttr) throws SQLException {
470 464
		Integer specTaxId = nullSafeInt(rs, specTaxIdAttr);
471 465
		if (specTaxId != null){
472 466
			String copyright = rs.getString(copyrightAttr);
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportBase.java
37 37
import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
38 38
import eu.etaxonomy.cdm.model.common.Language;
39 39
import eu.etaxonomy.cdm.model.common.User;
40
import eu.etaxonomy.cdm.model.location.NamedArea;
40 41
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
41 42
import eu.etaxonomy.cdm.model.name.ZoologicalName;
42 43
import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
......
58 59
	protected static final String REFERENCE_NAMESPACE = "Literatur";
59 60
	protected static final String TAXON_NAMESPACE = "current_species";
60 61
	protected static final String COLLECTION_NAMESPACE = "Collection";
62
	protected static final String IMAGE_NAMESPACE = "Einzelbilder";
63
	protected static final String SPEC_TAX_NAMESPACE = "specTax";
64
	protected static final String TYPE_NAMESPACE = "specTax.SpecTypeDepository";
61 65
	
62 66
	private String pluralString;
63 67
	private String dbTableName;
......
87 91

  
88 92
		int recordsPerTransaction = config.getRecordsPerTransaction();
89 93
		try{
90
			ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery, strRecordQuery, recordsPerTransaction);
94
			ResultSetPartitioner<GlobisImportState> partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery, strRecordQuery, recordsPerTransaction);
91 95
			while (partitioner.nextPartition()){
92 96
				partitioner.doPartition(this, state);
93 97
			}
......
104 108
	 * @param authorAndYear
105 109
	 * @param zooName
106 110
	 */
107
	protected void handleAuthorAndYear(String authorAndYear, ZoologicalName zooName) {
111
	protected void handleAuthorAndYear(String authorAndYear, ZoologicalName zooName, Integer id) {
108 112
		if (isBlank(authorAndYear)){
109 113
			return;
110 114
		}
......
117 121
			if (authorAndYear.contains("?")){
118 122
				authorAndYear = authorAndYear.replace("H?bner", "H\u00fcbner");
119 123
				authorAndYear = authorAndYear.replace("Oberth?r", "Oberth\u00fcr");
124
				authorAndYear = authorAndYear.replace("M?n?tri?s","M\u00E9n\u00E9tri\u00E9s");
125
				authorAndYear = authorAndYear.replace("Schifferm?ller","Schifferm\u00fcller");
126
				
120 127
				//TODO remove
121 128
				authorAndYear = authorAndYear.replace("?", "");
122 129
				
......
128 135
			}
129 136
			
130 137
		} catch (StringNotParsableException e) {
131
			logger.warn("Author could not be parsed: " + authorAndYear);
138
			logger.warn("Author could not be parsed: " + authorAndYear + " for id "  +id);
132 139
			zooName.setAuthorshipCache(authorAndYear, true);
133 140
		}
134 141
	}
......
139 146
	 * @param countryStr
140 147
	 * @return
141 148
	 */
142
	protected WaterbodyOrCountry getCountry(GlobisImportState state, String countryStr) {
143
		WaterbodyOrCountry country = WaterbodyOrCountry.getWaterbodyOrCountryByLabel(countryStr);
149
	protected NamedArea getCountry(GlobisImportState state, String countryStr) {
150
		NamedArea country = WaterbodyOrCountry.getWaterbodyOrCountryByLabel(countryStr);
144 151
		if (country == null){
145 152
			try {
146
				country = (WaterbodyOrCountry)state.getTransformer().getNamedAreaByKey(countryStr);
153
				country = (NamedArea)state.getTransformer().getNamedAreaByKey(countryStr);
147 154
			} catch (UndefinedTransformerMethodException e) {
148 155
				e.printStackTrace();
149 156
			}
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportConfigurator.java
60 60
	//			, ErmsReferenceImport.class
61 61
				, GlobisCurrentSpeciesImport.class
62 62
				, GlobisSpecTaxImport.class
63
				, GlobisCommonNameImport.class
63 64
				, GlobisImageImport.class
64 65
		};	
65 66
	}
......
162 163
		return doReadMediaData;
163 164
	}
164 165

  
165

  
166
	
167

  
168 166
}
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisSpecTaxImport.java
29 29
import eu.etaxonomy.cdm.io.common.IOValidator;
30 30
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
31 31
import eu.etaxonomy.cdm.io.common.mapping.IMappingImport;
32
import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
32 33
import eu.etaxonomy.cdm.io.globis.validation.GlobisSpecTaxaImportValidator;
33 34
import eu.etaxonomy.cdm.model.common.CdmBase;
34 35
import eu.etaxonomy.cdm.model.common.ExtensionType;
35 36
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
36 37
import eu.etaxonomy.cdm.model.common.Marker;
37 38
import eu.etaxonomy.cdm.model.common.MarkerType;
38
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
39
import eu.etaxonomy.cdm.model.description.Feature;
40
import eu.etaxonomy.cdm.model.location.NamedArea;
39 41
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
40 42
import eu.etaxonomy.cdm.model.name.Rank;
41 43
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
......
67 69
	private static final String pluralString = "taxa";
68 70
	private static final String dbTableName = "specTax";
69 71
	private static final Class cdmTargetClass = Reference.class;
70
	public static final String SPEC_TAX_NAMESPACE = dbTableName;
71
	public static final String TYPE_NAMESPACE = dbTableName + ".SpecTypeDepository";
72 72
	
73 73
	private static UUID uuidCitedTypeLocality = UUID.fromString("ca431e0a-84ec-4828-935f-df4c8f5cf880");
74 74
	private static UUID uuidCitedTypeMaterial = UUID.fromString("8395021a-e596-4a55-9794-8c03aaad9e16");
......
137 137
        		Integer acceptedTaxonId = nullSafeInt(rs, "SpecCurrspecID");
138 138
        		String specSystaxRank = rs.getString("SpecSystaxRank");
139 139
        		
140
        		//ignore: CountryDummy, currentSpecies, DepositoryDisplay, DepositoryDummy, ReferenceDisplay, SpecDescriptionImageFile, all *Valid*
141
        		
140 142
				try {
141 143
					
142 144
					//source ref
......
189 191
						objectsToSave.add(acceptedTaxon); 
190 192
					}
191 193
					
194
					//makeMarker1(state, rs, name);   //ignore!
195
					
196
					makeNotAvailable(state, rs, name);
197
					
192 198
					//SpecCitedTypeLocality
193 199
					String citedTypeLocality = rs.getString("SpecCitedTypeLocality");
194 200
					if (isNotBlank(citedTypeLocality)){
195
						ExtensionType exTypeCitedTypeLoc = getExtensionType(state, uuidCitedTypeLocality, "Type locality as cited in original description", "Type locality as cited in original description", null, ExtensionType.DOI().getVocabulary());
196
						name.addExtension(citedTypeLocality, exTypeCitedTypeLoc);
201
//						ExtensionType exTypeCitedTypeLoc = getExtensionType(state, uuidCitedTypeLocality, "Type locality as cited in original description", "Type locality as cited in original description", null, ExtensionType.DOI().getVocabulary());
202
//						name.addExtension(citedTypeLocality, exTypeCitedTypeLoc);
203
						addNameDescription(state, name, uuidCitedTypeLocality, citedTypeLocality, "Type locality as cited in original description");
197 204
					}
198 205

  
199 206
					//SpecCitedTypeMaterial
200 207
					String citedTypeMaterial = rs.getString("SpecCitedTypeMaterial");
201 208
					if (isNotBlank(citedTypeMaterial)){
202
						ExtensionType exTypeCitedTypeLoc = getExtensionType(state, uuidCitedTypeMaterial, "Type material as cited in original description", "Type locality as cited in original description", null, ExtensionType.DOI().getVocabulary());
209
						ExtensionType exTypeCitedTypeLoc = getExtensionType(state, uuidCitedTypeMaterial, "Type material as cited in original description", "Type material as cited in original description", null, ExtensionType.DOI().getVocabulary());
203 210
						name.addExtension(citedTypeLocality, exTypeCitedTypeLoc);
204 211
					}
205 212

  
213
					name.addSource(String.valueOf(specTaxId), SPEC_TAX_NAMESPACE, state.getTransactionalSourceReference(), null);
206 214
					
207 215
					namesToSave.add(name);
208 216
					
......
226 234
	}
227 235

  
228 236

  
237
	private void makeNotAvailable(GlobisImportState state, ResultSet rs, ZoologicalName name) throws SQLException {
238
		String notAvailableStr = rs.getString("SpecNotAvailable");
239
		try {
240
			if (isNotBlank(notAvailableStr)){
241
				if (notAvailableStr.contains("not available") ){ 
242
					UUID uuidNotAvailableMarkerType = state.getTransformer().getMarkerTypeUuid("not available");
243
					
244
					MarkerType markerType = getMarkerType(state, uuidNotAvailableMarkerType, "not available", "not available", null);
245
					name.addMarker(Marker.NewInstance(markerType, true));
246
				}
247
			}
248
		} catch (UndefinedTransformerMethodException e) {
249
			e.printStackTrace();
250
		}
251
		//Not available reason
252
		//TODO make it a vocabulary
253
		String notAvailableReason = rs.getString("SpecNotAvailableReason");
254
		if (isNotBlank(notAvailableReason)){
255
			UUID uuidNotAvailableReason;
256
			try {
257
				uuidNotAvailableReason = state.getTransformer().getExtensionTypeUuid("not available reason");
258
				ExtensionType notAvailableReasonExtType = getExtensionType(state, uuidNotAvailableReason, "Not available reason", "Not available reason", null, null);
259
				name.addExtension(notAvailableReason, notAvailableReasonExtType);
260
			} catch (UndefinedTransformerMethodException e) {
261
				e.printStackTrace();
262
			} 
263
		}
264
		
265
	}
266

  
267

  
268

  
269

  
270
	
271
	/**
272
	 * This method is not used anymore as according to Alexander Marker1 should be ignored.
273
	 * @param state
274
	 * @param rs
275
	 * @param name
276
	 * @throws SQLException
277
	 */
278
	private void makeMarker1(GlobisImportState state, ResultSet rs, ZoologicalName name) throws SQLException {
279
		String marker1Str = rs.getString("Marker1");
280
		try {
281
			if (isNotBlank(marker1Str)){
282
				marker1Str = marker1Str.trim();
283
				if (marker1Str.contains("checked") || marker1Str.contains("berpr") ){ //überprüft
284
					UUID uuidCheckedMarkerType;
285
						uuidCheckedMarkerType = state.getTransformer().getMarkerTypeUuid("checked");
286
					
287
					MarkerType markerType = getMarkerType(state, uuidCheckedMarkerType, "checked", "checked", null);
288
					name.addMarker(Marker.NewInstance(markerType, true));
289
				}
290
				if (marker1Str.contains("old record") || marker1Str.contains("alte Angabe") ){
291
					UUID uuidOldRecordMarkerType = state.getTransformer().getMarkerTypeUuid("old record");
292
					MarkerType markerType = getMarkerType(state, uuidOldRecordMarkerType, "checked", "checked", null);
293
					name.addMarker(Marker.NewInstance(markerType, true));
294
				}
295
			}
296
		} catch (UndefinedTransformerMethodException e) {
297
			e.printStackTrace();
298
		}
299
		
300
	}
301

  
302

  
303
	private void addNameDescription(GlobisImportState state, ZoologicalName name, UUID featureUuid,
304
			String citedTypeLocality, String featureLabel) {
305
		Feature feature = getFeature(state, featureUuid,featureLabel,featureLabel, null, null);
306
		getTaxonNameDescription(name, false, true);
307
		
308
	}
309

  
310

  
229 311
	private Pattern patternAll = Pattern.compile("(.+,\\s.+)(\\(.+\\))");
230 312
	
231 313

  
......
248 330
		//TODO several issues
249 331
		if (specTypeDepositories.length == 0){
250 332
			Specimen specimen = makeSingleTypeSpecimen(fieldObservation);
251
			makeTypeDesignation(name, rs, specimen);
333
			makeTypeDesignation(name, rs, specimen, specTaxId);
252 334
			makeTypeIdInSource(state, specimen, "null", specTaxId);
253 335
		}
254 336
		for (String specTypeDepositoryStr : specTypeDepositories){
......
278 360
			}
279 361
			
280 362
			//type Designation
281
			makeTypeDesignation(name, rs, specimen);
363
			makeTypeDesignation(name, rs, specimen, specTaxId);
282 364
		}
283 365

  
284 366
		
......
470 552
		DerivedUnitType unitType = DerivedUnitType.Specimen;
471 553
		DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(unitType);
472 554
		
473
		WaterbodyOrCountry typeCountry = getCountry(state, countryString);
555
		NamedArea typeCountry = getCountry(state, countryString);
474 556
		facade.setCountry(typeCountry);
475 557
		FieldObservation fieldObservation = facade.innerFieldObservation();
476 558
		return fieldObservation;
......
484 566
	 * @param rs 
485 567
	 * @param status
486 568
	 * @param specimen
569
	 * @param specTaxId 
487 570
	 * @throws SQLException 
488 571
	 */
489
	protected void makeTypeDesignation(ZoologicalName name, ResultSet rs, Specimen specimen) throws SQLException {
572
	protected void makeTypeDesignation(ZoologicalName name, ResultSet rs, Specimen specimen, Integer specTaxId) throws SQLException {
490 573
		//type
491 574
		String specType = rs.getString("SpecType");
492
		SpecimenTypeDesignationStatus status = getTypeDesigType(specType);
575
		SpecimenTypeDesignationStatus status = getTypeDesigType(specType, specTaxId);
493 576

  
494 577
		SpecimenTypeDesignation typeDesignation = SpecimenTypeDesignation.NewInstance();
495 578
		typeDesignation.setTypeStatus(status);
......
501 584

  
502 585

  
503 586

  
504
	private SpecimenTypeDesignationStatus getTypeDesigType(String specType) {
587
	private SpecimenTypeDesignationStatus getTypeDesigType(String specType, Integer specTaxId) {
505 588
		if (isBlank(specType) ){
506 589
			return null;
507
		}else if (specType.matches("Holotype(Holotypus)?")){
590
		}else if (specType.matches("Holotype(.*Holotypus)?")){
508 591
			return SpecimenTypeDesignationStatus.HOLOTYPE();
509 592
		}else if (specType.matches("Neotype")){
510 593
			return SpecimenTypeDesignationStatus.NEOTYPE();
......
513 596
		}else if (specType.matches("Lectotype")){
514 597
			return SpecimenTypeDesignationStatus.LECTOTYPE();
515 598
		}else{
516
			logger.warn("SpecimenTypeDesignationStatus does not match: " + specType);
599
			logger.warn("SpecimenTypeDesignationStatus does not match: " + specType + " in specTaxId "  + specTaxId);
517 600
			return null;
518 601
		}
519 602
	}
......
613 696
		String authorStr = rs.getString("SpecAuthor");
614 697
		String yearStr = rs.getString("SpecYear");
615 698
		String authorAndYearStr = CdmUtils.concat(", ", authorStr, yearStr);
616
		handleAuthorAndYear(authorAndYearStr, name);
699
		handleAuthorAndYear(authorAndYearStr, name, specTaxId);
617 700
		
618
		name.addSource(String.valueOf(specTaxId), SPEC_TAX_NAMESPACE, state.getTransactionalSourceReference(), null);
619 701
		return name;
620 702
	}
621 703

  
......
717 799

  
718 800

  
719 801
	@Override
720
	public Reference createObject(ResultSet rs, GlobisImportState state)
802
	public Reference<?> createObject(ResultSet rs, GlobisImportState state)
721 803
			throws SQLException {
722 804
		// not needed
723 805
		return null;
724 806
	}
725 807

  
726

  
727

  
728

  
729

  
730 808
}
app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisTransformer.java
22 22
import eu.etaxonomy.cdm.model.common.ExtensionType;
23 23
import eu.etaxonomy.cdm.model.description.Feature;
24 24
import eu.etaxonomy.cdm.model.location.NamedArea;
25
import eu.etaxonomy.cdm.model.location.TdwgArea;
25 26
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
26
import eu.etaxonomy.cdm.model.name.NameTypeDesignationStatus;
27 27

  
28 28
/**
29 29
 * @author a.mueller
......
34 34
	private static final Logger logger = Logger.getLogger(GlobisTransformer.class);
35 35
	
36 36

  
37
	//extension types
38
//	public static final UUID uuidEditor = UUID.fromString("07752659-3018-4880-bf26-41bb396fbf37");
39
	
40
	
41
	//language uuids
42
	
43
	
44
	
37
	//marker types
38
	public static final UUID uuidCheckedMarkerType = UUID.fromString("f2a7926f-def1-49a6-b642-9b81e6b1e35c");
39
	public static final UUID uuidOldRecordMarkerType = UUID.fromString("8616edc5-00d4-40ca-aca4-d48ec32231e9");
40
	public static final UUID uuidNotAvailableMarkerType = UUID.fromString("6931e584-6fc2-44ab-9084-e6452f8cd5d1");
45 41
	
42
	//extension types
43
	public static final UUID uuidExtTypeNotAvailableReason = UUID.fromString("d7dd5632-8c65-4058-b804-d1291560ac4c");
46 44
	
47
	/* (non-Javadoc)
48
	 * @see eu.etaxonomy.cdm.io.common.mapping.InputTransformerBase#getNameTypeDesignationStatusByKey(java.lang.String)
49
	 */
50
	@Override
51
	public NameTypeDesignationStatus getNameTypeDesignationStatusByKey(String key) throws UndefinedTransformerMethodException {
52
		if (key == null){
53
			return null;
54
		}
55
		Integer intDesignationId = Integer.valueOf(key);
56
		switch (intDesignationId){
57
			case 1: return NameTypeDesignationStatus.ORIGINAL_DESIGNATION();
58
			case 2: return NameTypeDesignationStatus.SUBSEQUENT_DESIGNATION();
59
			case 3: return NameTypeDesignationStatus.MONOTYPY();
60
			default: 
61
				String warning = "Unknown name type designation status id " + key;
62
				logger.warn(warning);
63
				return null;
64
		}
65
	}
66

  
67

  
68

  
69

  
70
	/* (non-Javadoc)
71
	 * @see eu.etaxonomy.cdm.io.common.mapping.InputTransformerBase#getNameTypeDesignationStatusUuid(java.lang.String)
72
	 */
73
	@Override
74
	public UUID getNameTypeDesignationStatusUuid(String key) throws UndefinedTransformerMethodException {
75
		//nott needed
76
		return super.getNameTypeDesignationStatusUuid(key);
77
	}
78

  
79

  
80 45
	public NamedArea getNamedAreaByKey(String area)  {
81 46
		Set<String> unhandledCountries = new HashSet<String>();
82 47
		
......
222 187
		}else if (area.equals("Dominica")){return WaterbodyOrCountry.DOMINICACOMMONWEALTHOF();
223 188
		}else if (area.equals("Liechtenstein")){return WaterbodyOrCountry.LIECHTENSTEINPRINCIPALITYOF();
224 189
		}else if (area.matches("B(y)?elarus")){return WaterbodyOrCountry.BELARUS();
225
		
226
		
227
		
228
		
190
		}else if (area.matches("United States: Alaska")){ return TdwgArea.getAreaByTdwgAbbreviation("ASK");
229 191
		
230 192
		}else{	
231 193
			if (unhandledCountries.contains(area)){
232
//				logger.warn("Unhandled country '" + area + "' replaced by null" );
194
				logger.warn("Unhandled country '" + area + "' replaced by null" );
233 195
				return null;
234 196
			}
235
//			String warning = "New language abbreviation " + area;
236
//			logger.warn(warning);
237 197
			return null;
238
//			throw new IllegalArgumentException(warning);
198

  
239 199
		}
240
		
241
		
242
		
200

  
243 201
	}
244 202
	
245 203
	/* (non-Javadoc)
......
273 231
	public UUID getExtensionTypeUuid(String key)
274 232
			throws UndefinedTransformerMethodException {
275 233
		if (key == null){return null;
276
//		}else if (key.equalsIgnoreCase("recent only")){return uuidRecentOnly;
234
		}else if (key.equalsIgnoreCase("not available reason")){return uuidExtTypeNotAvailableReason;
277 235
//		}else if (key.equalsIgnoreCase("recent + fossil")){return uuidRecentAndFossil;
278 236
//		}else if (key.equalsIgnoreCase("fossil only")){return uuidFossilOnly;
279 237
		}
280 238
		return null;
281 239
	}
240
	
241
	/* (non-Javadoc)
242
	 * @see eu.etaxonomy.cdm.io.common.mapping.InputTransformerBase#getExtensionTypeUuid(java.lang.String)
243
	 */
244
	@Override
245
	public UUID getMarkerTypeUuid(String key)
246
			throws UndefinedTransformerMethodException {
247
		if (key == null){return null;
248
		}else if (key.equalsIgnoreCase("old record")){return uuidOldRecordMarkerType;
249
		}else if (key.equalsIgnoreCase("checked")){return uuidCheckedMarkerType;
250
		}else if (key.equalsIgnoreCase("not available")){return uuidNotAvailableMarkerType;
251
		
252
		}
253
		return null;
254
	}
282 255

  
283 256
	
284 257

  

Also available in: Unified diff