reintegrate cdmlib-app-3.3 branch into trunk
[cdmlib-apps.git] / app-import / src / main / java / eu / etaxonomy / cdm / io / algaterra / AlgaTerraTypeImport.java
1 /**
2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
5 *
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
8 */
9
10 package eu.etaxonomy.cdm.io.algaterra;
11
12 import java.sql.ResultSet;
13 import java.sql.SQLException;
14 import java.util.HashMap;
15 import java.util.HashSet;
16 import java.util.Map;
17 import java.util.Set;
18
19 import org.apache.commons.lang.StringUtils;
20 import org.apache.log4j.Logger;
21 import org.springframework.stereotype.Component;
22
23 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeNotSupportedException;
25 import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraTypeImportValidator;
26 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
27 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState;
28 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport;
29 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonNameImport;
30 import eu.etaxonomy.cdm.io.common.IOValidator;
31 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
32 import eu.etaxonomy.cdm.model.common.CdmBase;
33 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
34 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
35 import eu.etaxonomy.cdm.model.name.BotanicalName;
36 import eu.etaxonomy.cdm.model.name.Rank;
37 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
38 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
39 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
40 import eu.etaxonomy.cdm.model.occurrence.Collection;
41 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
42 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
43 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
44 import eu.etaxonomy.cdm.model.reference.Reference;
45
46
47 /**
48 * @author a.mueller
49 * @created 20.03.2008
50 * @version 1.0
51 */
52 @Component
53 public class AlgaTerraTypeImport extends AlgaTerraSpecimenImportBase {
54 private static final Logger logger = Logger.getLogger(AlgaTerraTypeImport.class);
55
56
57 private static int modCount = 5000;
58 private static final String pluralString = "types";
59 private static final String dbTableName = "TypeDesignation"; //??
60
61 protected String getLocalityString(){
62 return "TypeLocality";
63 }
64
65 public AlgaTerraTypeImport(){
66 super(dbTableName, pluralString);
67 }
68
69
70 /* (non-Javadoc)
71 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery()
72 */
73 @Override
74 protected String getIdQuery(BerlinModelImportState state) {
75 String result = " SELECT TypeDesignationId "
76 + " FROM TypeDesignation "
77 + " ORDER BY NameFk ";
78 return result;
79 }
80
81 /* (non-Javadoc)
82 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
83 */
84 @Override
85 protected String getRecordQuery(BerlinModelImportConfigurator config) {
86 String strQuery =
87
88 " SELECT ts.*, ts.TypeSpecimenId as unitId, td.*, gz.ID as GazetteerId, gz.L2Code, gz.L3Code, gz.L4Code, gz.ISOCountry, gz.Country, ts.WaterBody, " +
89 " ts.RefFk as tsRefFk, ts.RefDetailFk as tsRefDetailFk, td.RefFk as tdRefFk, td.RefDetailFk as tdRefDetailFk " +
90 " FROM TypeSpecimenDesignation tsd "
91 + " LEFT OUTER JOIN TypeSpecimen AS ts ON tsd.TypeSpecimenFk = ts.TypeSpecimenId "
92 + " FULL OUTER JOIN TypeDesignation td ON td.TypeDesignationId = tsd.TypeDesignationFk "
93 + " LEFT OUTER JOIN TDWGGazetteer gz ON ts.TDWGGazetteerFk = gz.ID "
94 + " WHERE (td.TypeDesignationId IN (" + ID_LIST_TOKEN + ") )"
95 + " ORDER BY NameFk "
96 ;
97 return strQuery;
98 }
99
100 /* (non-Javadoc)
101 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
102 */
103 public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState bmState) {
104 boolean success = true;
105
106 AlgaTerraImportState state = (AlgaTerraImportState)bmState;
107
108 //TODO check that no duplicate vocabularies will be created, also remove redundant code here
109 // and in Specimen importer
110 try {
111 makeVocabulariesAndFeatures(state);
112 } catch (SQLException e1) {
113 logger.warn("Exception occurred when trying to create Type specimen vocabularies: " + e1.getMessage());
114 e1.printStackTrace();
115 }
116
117
118
119 Set<TaxonNameBase> namesToSave = new HashSet<TaxonNameBase>();
120
121 Map<String, TaxonNameBase> taxonNameMap = (Map<String, TaxonNameBase>) partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);
122 Map<String, DerivedUnit> ecoFactMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(AlgaTerraEcoFactImport.ECO_FACT_FIELD_OBSERVATION_NAMESPACE);
123 Map<String, DerivedUnit> typeSpecimenMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE);
124 Map<String, Reference> biblioRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE);
125 Map<String, Reference> nomRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE);
126
127
128 ResultSet rs = partitioner.getResultSet();
129
130 try {
131
132 int i = 0;
133
134 //for each reference
135 while (rs.next()){
136
137 if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("Type designations handled: " + (i-1));}
138
139 int nameId = rs.getInt("nameFk");
140 int typeSpecimenId = rs.getInt("TypeSpecimenId");
141 int typeDesignationId = rs.getInt("TypeDesignationId");
142 Integer typeStatusFk = nullSafeInt(rs, "typeStatusFk");
143 Integer ecoFactId = nullSafeInt(rs, "ecoFactFk");
144 Integer tdRefFk = nullSafeInt(rs, "tdRefFk");
145 Integer tdRefDetailFk = nullSafeInt(rs, "tdRefDetailFk");
146
147
148 // String recordBasis = rs.getString("RecordBasis");
149
150 try {
151
152 //source ref
153 Reference<?> sourceRef = state.getTransactionalSourceReference();
154
155 //facade
156 //FIXME - depends on material category
157 // DerivedUnitType type = makeDerivedUnitType(recordBasis);
158 if (i<=2){logger.warn("RecordBasis not yet implemented in TypeImport");}
159
160 SpecimenOrObservationType type = SpecimenOrObservationType.PreservedSpecimen;
161 DerivedUnitFacade facade = getDerivedUnit(state, typeSpecimenId, typeSpecimenMap, type, ecoFactMap, ecoFactId);
162
163 //field observation
164 handleFieldObservationSpecimen(rs, facade, state, partitioner);
165
166 //TODO divide like in EcoFact (if necessary)
167 handleTypeSpecimenSpecificSpecimen(rs,facade, state, biblioRefMap, nomRefMap, typeSpecimenId);
168
169 handleFirstDerivedSpecimen(rs, facade, state, partitioner);
170
171
172 //Designation
173 TaxonNameBase<?,?> name = getTaxonName(state, taxonNameMap, nameId);
174 SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
175 SpecimenTypeDesignationStatus status = getSpecimenTypeDesignationStatusByKey(typeStatusFk);
176 designation.setTypeSpecimen(facade.innerDerivedUnit());
177 designation.setTypeStatus(status);
178 if (tdRefFk != null){
179 Reference<?> typeDesigRef = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(tdRefFk));
180 if (typeDesigRef == null){
181 logger.warn("Type designation reference not found in maps: " + tdRefFk);
182 }else{
183 designation.setCitation(typeDesigRef);
184 }
185 }
186
187 if (tdRefDetailFk != null){
188 logger.warn("TypeDesignation.RefDetailFk not yet implemented: " + typeDesignationId);
189 }
190
191 if (name != null){
192 name.addTypeDesignation(designation, true); //TODO check if true is correct
193 }else{
194 logger.warn("Name could not be found for type designation " + typeDesignationId);
195 }
196 namesToSave.add(name);
197
198
199 } catch (Exception e) {
200 logger.warn("Exception in TypeDesignation: TypeDesignationId " + typeDesignationId + ". " + e.getMessage());
201 e.printStackTrace();
202 }
203
204 }
205
206 // logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
207
208 logger.warn("Names to save: " + namesToSave.size());
209 getNameService().save(namesToSave);
210
211 return success;
212 } catch (SQLException e) {
213 logger.error("SQLException:" + e);
214 return false;
215 }
216 }
217
218
219 protected String getDerivedUnitNameSpace(){
220 return TYPE_SPECIMEN_DERIVED_UNIT_NAMESPACE;
221 }
222
223 protected String getFieldObservationNameSpace(){
224 return TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE;
225 }
226
227 /**
228 * @param state
229 * @param taxonNameMap
230 * @param nameId
231 * @return
232 */
233 private TaxonNameBase<?,?> getTaxonName(AlgaTerraImportState state, Map<String, TaxonNameBase> taxonNameMap, int nameId) {
234 TaxonNameBase<?,?> result;
235 if (state.getConfig().isDoTaxonNames()){
236 result = taxonNameMap.get(String.valueOf(nameId));
237 }else{
238 //for testing
239 result = BotanicalName.NewInstance(Rank.SPECIES());
240 }
241 return result;
242 }
243
244 private void handleTypeSpecimenSpecificSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state, Map<String, Reference> biblioRefMap, Map<String, Reference> nomRefMap, int typeSpecimenId) throws SQLException {
245
246
247 //TODO
248
249
250
251 DerivedUnit derivedUnit = facade.innerDerivedUnit();
252
253 //collection
254 String barcode = rs.getString("Barcode");
255 if (StringUtils.isNotBlank(barcode)){
256 facade.setBarcode(barcode);
257 }
258
259 //RefFk + RefDetailFk
260 Integer refFk = nullSafeInt(rs, "tsRefFk");
261 if (refFk != null){
262
263 Reference<?> ref = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(refFk));
264 if (ref == null){
265 logger.warn("TypeSpecimen reference (" + refFk + ")not found in biblioRef. TypeSpecimenId: " + typeSpecimenId);
266 }else{
267 IdentifiableSource source = IdentifiableSource.NewPrimarySourceInstance(ref, null);
268 derivedUnit.addSource(source);
269 }
270 }
271
272 Integer refDetailFk = nullSafeInt(rs, "tsRefDetailFk");
273 if (refDetailFk != null){
274 logger.warn("TypeSpecimen.RefDetailFk should always be NULL but wasn't: " + typeSpecimenId);
275 }
276
277
278
279 }
280
281 /**
282 * @param state
283 * @param ecoFactId
284 * @param derivedUnitMap
285 * @param type
286 * @param ecoFactId2
287 * @param ecoFactMap
288 * @return
289 */
290 private DerivedUnitFacade getDerivedUnit(AlgaTerraImportState state, int typeSpecimenId, Map<String, DerivedUnit> typeSpecimenMap, SpecimenOrObservationType type, Map<String, DerivedUnit> ecoFactMap, Integer ecoFactId2) {
291 //TODO implement ecoFact map - if not all null anymore
292 String typeKey = String.valueOf(typeSpecimenId);
293 DerivedUnit derivedUnit = typeSpecimenMap.get(typeKey);
294 DerivedUnitFacade facade;
295 if (derivedUnit == null){
296 facade = DerivedUnitFacade.NewInstance(type);
297 typeSpecimenMap.put(typeKey, derivedUnit);
298 }else{
299 try {
300 facade = DerivedUnitFacade.NewInstance(derivedUnit);
301 } catch (DerivedUnitFacadeNotSupportedException e) {
302 logger.error(e.getMessage());
303 facade = DerivedUnitFacade.NewInstance(type);
304 }
305 }
306
307 return facade;
308 }
309 // private DerivedUnitType makeDerivedUnitType(String recordBasis) {
310 // DerivedUnitType result = null;
311 // if (StringUtils.isBlank(recordBasis)){
312 // result = DerivedUnitType.DerivedUnit;
313 // } else if (recordBasis.equalsIgnoreCase("FossileSpecimen")){
314 // result = DerivedUnitType.Fossil;
315 // }else if (recordBasis.equalsIgnoreCase("HumanObservation")){
316 // result = DerivedUnitType.Observation;
317 // }else if (recordBasis.equalsIgnoreCase("Literature")){
318 // logger.warn("Literature record basis not yet supported");
319 // result = DerivedUnitType.DerivedUnit;
320 // }else if (recordBasis.equalsIgnoreCase("LivingSpecimen")){
321 // result = DerivedUnitType.LivingBeing;
322 // }else if (recordBasis.equalsIgnoreCase("MachineObservation")){
323 // logger.warn("MachineObservation record basis not yet supported");
324 // result = DerivedUnitType.Observation;
325 // }else if (recordBasis.equalsIgnoreCase("PreservedSpecimen")){
326 // result = DerivedUnitType.Specimen;
327 // }
328 // return result;
329 // }
330
331
332 private SpecimenTypeDesignationStatus getSpecimenTypeDesignationStatusByKey(Integer typeStatusFk) {
333 if (typeStatusFk == null){ return null;
334 }else if (typeStatusFk == 1) { return SpecimenTypeDesignationStatus.HOLOTYPE();
335 }else if (typeStatusFk == 2) { return SpecimenTypeDesignationStatus.LECTOTYPE();
336 }else if (typeStatusFk == 3) { return SpecimenTypeDesignationStatus.NEOTYPE();
337 }else if (typeStatusFk == 4) { return SpecimenTypeDesignationStatus.EPITYPE();
338 }else if (typeStatusFk == 5) { return SpecimenTypeDesignationStatus.ISOLECTOTYPE();
339 }else if (typeStatusFk == 6) { return SpecimenTypeDesignationStatus.ISONEOTYPE();
340 }else if (typeStatusFk == 7) { return SpecimenTypeDesignationStatus.ISOTYPE();
341 }else if (typeStatusFk == 8) { return SpecimenTypeDesignationStatus.PARANEOTYPE();
342 }else if (typeStatusFk == 9) { return SpecimenTypeDesignationStatus.PARATYPE();
343 }else if (typeStatusFk == 10) { return SpecimenTypeDesignationStatus.SECOND_STEP_LECTOTYPE();
344 }else if (typeStatusFk == 11) { return SpecimenTypeDesignationStatus.SECOND_STEP_NEOTYPE();
345 }else if (typeStatusFk == 12) { return SpecimenTypeDesignationStatus.SYNTYPE();
346 }else if (typeStatusFk == 13) { return SpecimenTypeDesignationStatus.PARALECTOTYPE();
347 }else if (typeStatusFk == 14) { return SpecimenTypeDesignationStatus.ISOEPITYPE();
348 }else if (typeStatusFk == 21) { return SpecimenTypeDesignationStatus.ICONOTYPE();
349 }else if (typeStatusFk == 22) { return SpecimenTypeDesignationStatus.PHOTOTYPE();
350 }else if (typeStatusFk == 30) { return SpecimenTypeDesignationStatus.TYPE();
351 }else if (typeStatusFk == 38) { return SpecimenTypeDesignationStatus.ISOEPITYPE();
352 // }else if (typeStatusFk == 39) { return SpecimenTypeDesignationStatus.;
353 }else if (typeStatusFk == 40) { return SpecimenTypeDesignationStatus.ORIGINAL_MATERIAL();
354 }else{
355 logger.warn("typeStatusFk undefined for " + typeStatusFk);
356 return SpecimenTypeDesignationStatus.TYPE();
357 }
358
359 }
360
361
362 /* (non-Javadoc)
363 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
364 */
365 public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
366 String nameSpace;
367 Class cdmClass;
368 Set<String> idSet;
369 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
370
371 try{
372 Set<String> nameIdSet = new HashSet<String>();
373 Set<String> ecoFieldObservationIdSet = new HashSet<String>();
374 Set<String> typeSpecimenIdSet = new HashSet<String>();
375 Set<String> termsIdSet = new HashSet<String>();
376 Set<String> collectionIdSet = new HashSet<String>();
377 Set<String> referenceIdSet = new HashSet<String>();
378
379 while (rs.next()){
380 handleForeignKey(rs, nameIdSet, "nameFk");
381 handleForeignKey(rs, ecoFieldObservationIdSet, "ecoFactFk");
382 handleForeignKey(rs, typeSpecimenIdSet, "TypeSpecimenId");
383 handleForeignKey(rs, collectionIdSet, "CollectionFk");
384 handleForeignKey(rs, referenceIdSet, "tsRefFk");
385 handleForeignKey(rs, referenceIdSet, "tdRefFk");
386 }
387
388 //name map
389 nameSpace = BerlinModelTaxonNameImport.NAMESPACE;
390 cdmClass = TaxonNameBase.class;
391 idSet = nameIdSet;
392 Map<String, TaxonNameBase> objectMap = (Map<String, TaxonNameBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
393 result.put(nameSpace, objectMap);
394
395 //eco fact field observation map
396 nameSpace = AlgaTerraTypeImport.ECO_FACT_FIELD_OBSERVATION_NAMESPACE;
397 cdmClass = FieldUnit.class;
398 idSet = ecoFieldObservationIdSet;
399 Map<String, FieldUnit> fieldObservationMap = (Map<String, FieldUnit>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
400 result.put(nameSpace, fieldObservationMap);
401
402 //type specimen map
403 nameSpace = AlgaTerraTypeImport.TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE;
404 cdmClass = FieldUnit.class;
405 idSet = typeSpecimenIdSet;
406 Map<String, FieldUnit> typeSpecimenMap = (Map<String, FieldUnit>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
407 result.put(nameSpace, typeSpecimenMap);
408
409
410 //collections
411 nameSpace = AlgaTerraCollectionImport.NAMESPACE_COLLECTION;
412 cdmClass = Collection.class;
413 idSet = collectionIdSet;
414 Map<String, Collection> collectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
415 result.put(nameSpace, collectionMap);
416
417 //sub-collections
418 nameSpace = AlgaTerraCollectionImport.NAMESPACE_SUBCOLLECTION;
419 cdmClass = Collection.class;
420 idSet = collectionIdSet;
421 Map<String, Collection> subCollectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
422 result.put(nameSpace, subCollectionMap);
423
424 //nom reference map
425 nameSpace = BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE;
426 cdmClass = Reference.class;
427 idSet = referenceIdSet;
428 Map<String, Reference> nomReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
429 result.put(nameSpace, nomReferenceMap);
430
431 //biblio reference map
432 nameSpace = BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE;
433 cdmClass = Reference.class;
434 idSet = referenceIdSet;
435 Map<String, Reference> biblioReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
436 result.put(nameSpace, biblioReferenceMap);
437
438
439 //
440 // //terms
441 // nameSpace = AlgaTerraTypeImport.TERMS_NAMESPACE;
442 // cdmClass = FieldObservation.class;
443 // idSet = taxonIdSet;
444 // Map<String, DefinedTermBase> termMap = (Map<String, DefinedTermBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
445 // result.put(nameSpace, termMap);
446
447
448
449
450 } catch (SQLException e) {
451 throw new RuntimeException(e);
452 }
453 return result;
454 }
455
456 /* (non-Javadoc)
457 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
458 */
459 @Override
460 protected boolean doCheck(BerlinModelImportState state){
461 IOValidator<BerlinModelImportState> validator = new AlgaTerraTypeImportValidator();
462 return validator.validate(state);
463 }
464
465
466 /* (non-Javadoc)
467 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
468 */
469 protected boolean isIgnore(BerlinModelImportState state){
470 return ! state.getConfig().isDoTypes();
471 }
472
473 }