latest AlgaTerra import updates
[cdmlib-apps.git] / app-import / src / main / java / eu / etaxonomy / cdm / io / algaterra / AlgaTerraTypeImport.java
1 /**
2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
5 *
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
8 */
9
10 package eu.etaxonomy.cdm.io.algaterra;
11
12 import java.sql.ResultSet;
13 import java.sql.SQLException;
14 import java.util.HashMap;
15 import java.util.HashSet;
16 import java.util.Map;
17 import java.util.Set;
18
19 import org.apache.commons.lang.StringUtils;
20 import org.apache.log4j.Logger;
21 import org.springframework.stereotype.Component;
22
23 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
25 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeNotSupportedException;
26 import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraTypeImportValidator;
27 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
28 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState;
29 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelReferenceImport;
30 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonNameImport;
31 import eu.etaxonomy.cdm.io.common.IOValidator;
32 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
33 import eu.etaxonomy.cdm.model.common.CdmBase;
34 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
35 import eu.etaxonomy.cdm.model.name.BotanicalName;
36 import eu.etaxonomy.cdm.model.name.Rank;
37 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
38 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
39 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
40 import eu.etaxonomy.cdm.model.occurrence.Collection;
41 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
42 import eu.etaxonomy.cdm.model.occurrence.DerivedUnitBase;
43 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
44 import eu.etaxonomy.cdm.model.reference.Reference;
45
46
47 /**
48 * @author a.mueller
49 * @created 20.03.2008
50 * @version 1.0
51 */
52 @Component
53 public class AlgaTerraTypeImport extends AlgaTerraSpecimenImportBase {
54 private static final Logger logger = Logger.getLogger(AlgaTerraTypeImport.class);
55
56
57 private static int modCount = 5000;
58 private static final String pluralString = "types";
59 private static final String dbTableName = "TypeDesignation"; //??
60
61 protected String getLocalityString(){
62 return "TypeLocality";
63 }
64
65 public AlgaTerraTypeImport(){
66 super(dbTableName, pluralString);
67 }
68
69
70 /* (non-Javadoc)
71 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery()
72 */
73 @Override
74 protected String getIdQuery(BerlinModelImportState state) {
75 String result = " SELECT TypeDesignationId "
76 + " FROM TypeDesignation "
77 + " ORDER BY NameFk ";
78 return result;
79 }
80
81 /* (non-Javadoc)
82 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
83 */
84 @Override
85 protected String getRecordQuery(BerlinModelImportConfigurator config) {
86 String strQuery =
87
88 " SELECT ts.*, ts.TypeSpecimenId as unitId, td.*, gz.ID as GazetteerId, gz.L2Code, gz.L3Code, gz.L4Code, gz.ISOCountry, gz.Country, ts.WaterBody, " +
89 " ts.RefFk as tsRefFk, ts.RefDetailFk as tsRefDetailFk, td.RefFk as tdRefFk, td.RefDetailFk as tdRefDetailFk " +
90 " FROM TypeSpecimenDesignation tsd "
91 + " LEFT OUTER JOIN TypeSpecimen AS ts ON tsd.TypeSpecimenFk = ts.TypeSpecimenId "
92 + " FULL OUTER JOIN TypeDesignation td ON td.TypeDesignationId = tsd.TypeDesignationFk "
93 + " LEFT OUTER JOIN TDWGGazetteer gz ON ts.TDWGGazetteerFk = gz.ID "
94 + " WHERE (td.TypeDesignationId IN (" + ID_LIST_TOKEN + ") )"
95 + " ORDER BY NameFk "
96 ;
97 return strQuery;
98 }
99
100 /* (non-Javadoc)
101 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
102 */
103 public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState bmState) {
104 boolean success = true;
105
106 AlgaTerraImportState state = (AlgaTerraImportState)bmState;
107
108 //TODO check that no duplicate vocabularies will be created, also remove redundant code here
109 // and in Specimen importer
110 try {
111 makeVocabulariesAndFeatures(state);
112 } catch (SQLException e1) {
113 logger.warn("Exception occurred when trying to create Type specimen vocabularies: " + e1.getMessage());
114 e1.printStackTrace();
115 }
116
117
118
119 Set<TaxonNameBase> namesToSave = new HashSet<TaxonNameBase>();
120
121 Map<String, TaxonNameBase> taxonNameMap = (Map<String, TaxonNameBase>) partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);
122 Map<String, DerivedUnit> ecoFactMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(AlgaTerraEcoFactImport.ECO_FACT_FIELD_OBSERVATION_NAMESPACE);
123 Map<String, DerivedUnit> typeSpecimenMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE);
124 Map<String, Reference> biblioRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE);
125 Map<String, Reference> nomRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE);
126
127
128 ResultSet rs = partitioner.getResultSet();
129
130 try {
131
132 int i = 0;
133
134 //for each reference
135 while (rs.next()){
136
137 if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("Type designations handled: " + (i-1));}
138
139 int nameId = rs.getInt("nameFk");
140 int typeSpecimenId = rs.getInt("TypeSpecimenId");
141 int typeDesignationId = rs.getInt("TypeDesignationId");
142 Integer typeStatusFk = nullSafeInt(rs, "typeStatusFk");
143 Integer ecoFactId = nullSafeInt(rs, "ecoFactFk");
144 Integer tdRefFk = nullSafeInt(rs, "tdRefFk");
145 Integer tdRefDetailFk = nullSafeInt(rs, "tdRefDetailFk");
146
147
148 // String recordBasis = rs.getString("RecordBasis");
149
150 try {
151
152 //source ref
153 Reference<?> sourceRef = state.getTransactionalSourceReference();
154
155 //facade
156 //FIXME - depends on material category
157 // DerivedUnitType type = makeDerivedUnitType(recordBasis);
158 DerivedUnitType type = DerivedUnitType.Specimen;
159 DerivedUnitFacade facade = getDerivedUnit(state, typeSpecimenId, typeSpecimenMap, type, ecoFactMap, ecoFactId);
160
161 //field observation
162 handleFieldObservationSpecimen(rs, facade, state, partitioner);
163
164 //TODO divide like in EcoFact (if necessary)
165 handleTypeSpecimenSpecificSpecimen(rs,facade, state, biblioRefMap, nomRefMap, typeSpecimenId);
166
167 handleFirstDerivedSpecimen(rs, facade, state, partitioner);
168
169
170 //Designation
171 TaxonNameBase<?,?> name = getTaxonName(state, taxonNameMap, nameId);
172 SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
173 SpecimenTypeDesignationStatus status = getSpecimenTypeDesignationStatusByKey(typeStatusFk);
174 designation.setTypeSpecimen(facade.innerDerivedUnit());
175 designation.setTypeStatus(status);
176 if (tdRefFk != null){
177 Reference<?> typeDesigRef = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(tdRefFk));
178 if (typeDesigRef == null){
179 logger.warn("Type designation reference not found in maps: " + tdRefFk);
180 }else{
181 designation.setCitation(typeDesigRef);
182 }
183 }
184
185 if (tdRefDetailFk != null){
186 logger.warn("TypeDesignation.RefDetailFk not yet implemented: " + typeDesignationId);
187 }
188
189 if (name != null){
190 name.addTypeDesignation(designation, true); //TODO check if true is correct
191 }else{
192 logger.warn("Name could not be found for type designation " + typeDesignationId);
193 }
194 namesToSave.add(name);
195
196
197 } catch (Exception e) {
198 logger.warn("Exception in TypeDesignation: TypeDesignationId " + typeDesignationId + ". " + e.getMessage());
199 e.printStackTrace();
200 }
201
202 }
203
204 // logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
205
206 logger.warn("Names to save: " + namesToSave.size());
207 getNameService().save(namesToSave);
208
209 return success;
210 } catch (SQLException e) {
211 logger.error("SQLException:" + e);
212 return false;
213 }
214 }
215
216
217 protected String getDerivedUnitNameSpace(){
218 return TYPE_SPECIMEN_DERIVED_UNIT_NAMESPACE;
219 }
220
221 protected String getFieldObservationNameSpace(){
222 return TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE;
223 }
224
225 /**
226 * @param state
227 * @param taxonNameMap
228 * @param nameId
229 * @return
230 */
231 private TaxonNameBase<?,?> getTaxonName(AlgaTerraImportState state, Map<String, TaxonNameBase> taxonNameMap, int nameId) {
232 TaxonNameBase<?,?> result;
233 if (state.getConfig().isDoTaxonNames()){
234 result = taxonNameMap.get(String.valueOf(nameId));
235 }else{
236 //for testing
237 result = BotanicalName.NewInstance(Rank.SPECIES());
238 }
239 return result;
240 }
241
242 private void handleTypeSpecimenSpecificSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state, Map<String, Reference> biblioRefMap, Map<String, Reference> nomRefMap, int typeSpecimenId) throws SQLException {
243
244
245 //TODO
246
247
248
249 DerivedUnitBase<?> derivedUnit = facade.innerDerivedUnit();
250
251 //collection
252 String barcode = rs.getString("Barcode");
253 if (StringUtils.isNotBlank(barcode)){
254 facade.setBarcode(barcode);
255 }
256
257 //RefFk + RefDetailFk
258 Integer refFk = nullSafeInt(rs, "tsRefFk");
259 if (refFk != null){
260
261 Reference<?> ref = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(refFk));
262 if (ref == null){
263 logger.warn("TypeSpecimen reference (" + refFk + ")not found in biblioRef. TypeSpecimenId: " + typeSpecimenId);
264 }else{
265 IdentifiableSource source = IdentifiableSource.NewInstance(ref, null);
266 derivedUnit.addSource(source);
267 }
268 }
269
270 Integer refDetailFk = nullSafeInt(rs, "tsRefDetailFk");
271 if (refDetailFk != null){
272 logger.warn("TypeSpecimen.RefDetailFk should always be NULL but wasn't: " + typeSpecimenId);
273 }
274
275
276
277 }
278
279 /**
280 * @param state
281 * @param ecoFactId
282 * @param derivedUnitMap
283 * @param type
284 * @param ecoFactId2
285 * @param ecoFactMap
286 * @return
287 */
288 private DerivedUnitFacade getDerivedUnit(AlgaTerraImportState state, int typeSpecimenId, Map<String, DerivedUnit> typeSpecimenMap, DerivedUnitType type, Map<String, DerivedUnit> ecoFactMap, Integer ecoFactId2) {
289 //TODO implement ecoFact map - if not all null anymore
290 String typeKey = String.valueOf(typeSpecimenId);
291 DerivedUnit derivedUnit = typeSpecimenMap.get(typeKey);
292 DerivedUnitFacade facade;
293 if (derivedUnit == null){
294 facade = DerivedUnitFacade.NewInstance(type);
295 typeSpecimenMap.put(typeKey, derivedUnit);
296 }else{
297 try {
298 facade = DerivedUnitFacade.NewInstance(derivedUnit);
299 } catch (DerivedUnitFacadeNotSupportedException e) {
300 logger.error(e.getMessage());
301 facade = DerivedUnitFacade.NewInstance(type);
302 }
303 }
304
305 return facade;
306 }
307 private DerivedUnitType makeDerivedUnitType(String recordBasis) {
308 DerivedUnitType result = null;
309 if (StringUtils.isBlank(recordBasis)){
310 result = DerivedUnitType.DerivedUnit;
311 } else if (recordBasis.equalsIgnoreCase("FossileSpecimen")){
312 result = DerivedUnitType.Fossil;
313 }else if (recordBasis.equalsIgnoreCase("HumanObservation")){
314 result = DerivedUnitType.Observation;
315 }else if (recordBasis.equalsIgnoreCase("Literature")){
316 logger.warn("Literature record basis not yet supported");
317 result = DerivedUnitType.DerivedUnit;
318 }else if (recordBasis.equalsIgnoreCase("LivingSpecimen")){
319 result = DerivedUnitType.LivingBeing;
320 }else if (recordBasis.equalsIgnoreCase("MachineObservation")){
321 logger.warn("MachineObservation record basis not yet supported");
322 result = DerivedUnitType.Observation;
323 }else if (recordBasis.equalsIgnoreCase("PreservedSpecimen")){
324 result = DerivedUnitType.Specimen;
325 }
326 return result;
327 }
328
329
330 private SpecimenTypeDesignationStatus getSpecimenTypeDesignationStatusByKey(Integer typeStatusFk) {
331 if (typeStatusFk == null){ return null;
332 }else if (typeStatusFk == 1) { return SpecimenTypeDesignationStatus.HOLOTYPE();
333 }else if (typeStatusFk == 2) { return SpecimenTypeDesignationStatus.LECTOTYPE();
334 }else if (typeStatusFk == 3) { return SpecimenTypeDesignationStatus.NEOTYPE();
335 }else if (typeStatusFk == 4) { return SpecimenTypeDesignationStatus.EPITYPE();
336 }else if (typeStatusFk == 5) { return SpecimenTypeDesignationStatus.ISOLECTOTYPE();
337 }else if (typeStatusFk == 6) { return SpecimenTypeDesignationStatus.ISONEOTYPE();
338 }else if (typeStatusFk == 7) { return SpecimenTypeDesignationStatus.ISOTYPE();
339 }else if (typeStatusFk == 8) { return SpecimenTypeDesignationStatus.PARANEOTYPE();
340 }else if (typeStatusFk == 9) { return SpecimenTypeDesignationStatus.PARATYPE();
341 }else if (typeStatusFk == 10) { return SpecimenTypeDesignationStatus.SECOND_STEP_LECTOTYPE();
342 }else if (typeStatusFk == 11) { return SpecimenTypeDesignationStatus.SECOND_STEP_NEOTYPE();
343 }else if (typeStatusFk == 12) { return SpecimenTypeDesignationStatus.SYNTYPE();
344 }else if (typeStatusFk == 13) { return SpecimenTypeDesignationStatus.PARALECTOTYPE();
345 }else if (typeStatusFk == 14) { return SpecimenTypeDesignationStatus.ISOEPITYPE();
346 }else if (typeStatusFk == 21) { return SpecimenTypeDesignationStatus.ICONOTYPE();
347 }else if (typeStatusFk == 22) { return SpecimenTypeDesignationStatus.PHOTOTYPE();
348 }else if (typeStatusFk == 30) { return SpecimenTypeDesignationStatus.TYPE();
349 }else if (typeStatusFk == 38) { return SpecimenTypeDesignationStatus.ISOEPITYPE();
350 // }else if (typeStatusFk == 39) { return SpecimenTypeDesignationStatus.;
351 }else if (typeStatusFk == 40) { return SpecimenTypeDesignationStatus.ORIGINAL_MATERIAL();
352 }else{
353 logger.warn("typeStatusFk undefined for " + typeStatusFk);
354 return SpecimenTypeDesignationStatus.TYPE();
355 }
356
357 }
358
359
360 /* (non-Javadoc)
361 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
362 */
363 public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
364 String nameSpace;
365 Class cdmClass;
366 Set<String> idSet;
367 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
368
369 try{
370 Set<String> nameIdSet = new HashSet<String>();
371 Set<String> ecoFieldObservationIdSet = new HashSet<String>();
372 Set<String> typeSpecimenIdSet = new HashSet<String>();
373 Set<String> termsIdSet = new HashSet<String>();
374 Set<String> collectionIdSet = new HashSet<String>();
375 Set<String> referenceIdSet = new HashSet<String>();
376
377 while (rs.next()){
378 handleForeignKey(rs, nameIdSet, "nameFk");
379 handleForeignKey(rs, ecoFieldObservationIdSet, "ecoFactFk");
380 handleForeignKey(rs, typeSpecimenIdSet, "TypeSpecimenId");
381 handleForeignKey(rs, collectionIdSet, "CollectionFk");
382 handleForeignKey(rs, referenceIdSet, "tsRefFk");
383 handleForeignKey(rs, referenceIdSet, "tdRefFk");
384 }
385
386 //name map
387 nameSpace = BerlinModelTaxonNameImport.NAMESPACE;
388 cdmClass = TaxonNameBase.class;
389 idSet = nameIdSet;
390 Map<String, TaxonNameBase> objectMap = (Map<String, TaxonNameBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
391 result.put(nameSpace, objectMap);
392
393 //eco fact field observation map
394 nameSpace = AlgaTerraTypeImport.ECO_FACT_FIELD_OBSERVATION_NAMESPACE;
395 cdmClass = FieldObservation.class;
396 idSet = ecoFieldObservationIdSet;
397 Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
398 result.put(nameSpace, fieldObservationMap);
399
400 //type specimen map
401 nameSpace = AlgaTerraTypeImport.TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE;
402 cdmClass = FieldObservation.class;
403 idSet = typeSpecimenIdSet;
404 Map<String, FieldObservation> typeSpecimenMap = (Map<String, FieldObservation>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
405 result.put(nameSpace, typeSpecimenMap);
406
407
408 //collections
409 nameSpace = AlgaTerraCollectionImport.NAMESPACE_COLLECTION;
410 cdmClass = Collection.class;
411 idSet = collectionIdSet;
412 Map<String, Collection> collectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
413 result.put(nameSpace, collectionMap);
414
415 //sub-collections
416 nameSpace = AlgaTerraCollectionImport.NAMESPACE_SUBCOLLECTION;
417 cdmClass = Collection.class;
418 idSet = collectionIdSet;
419 Map<String, Collection> subCollectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
420 result.put(nameSpace, subCollectionMap);
421
422 //nom reference map
423 nameSpace = BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE;
424 cdmClass = Reference.class;
425 idSet = referenceIdSet;
426 Map<String, Reference> nomReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
427 result.put(nameSpace, nomReferenceMap);
428
429 //biblio reference map
430 nameSpace = BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE;
431 cdmClass = Reference.class;
432 idSet = referenceIdSet;
433 Map<String, Reference> biblioReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
434 result.put(nameSpace, biblioReferenceMap);
435
436
437 //
438 // //terms
439 // nameSpace = AlgaTerraTypeImport.TERMS_NAMESPACE;
440 // cdmClass = FieldObservation.class;
441 // idSet = taxonIdSet;
442 // Map<String, DefinedTermBase> termMap = (Map<String, DefinedTermBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
443 // result.put(nameSpace, termMap);
444
445
446
447
448 } catch (SQLException e) {
449 throw new RuntimeException(e);
450 }
451 return result;
452 }
453
454 /* (non-Javadoc)
455 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
456 */
457 @Override
458 protected boolean doCheck(BerlinModelImportState state){
459 IOValidator<BerlinModelImportState> validator = new AlgaTerraTypeImportValidator();
460 return validator.validate(state);
461 }
462
463
464 /* (non-Javadoc)
465 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
466 */
467 protected boolean isIgnore(BerlinModelImportState state){
468 return ! state.getConfig().isDoTypes();
469 }
470
471 }