2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.algaterra
;
12 import java
.sql
.ResultSet
;
13 import java
.sql
.SQLException
;
14 import java
.util
.HashMap
;
15 import java
.util
.HashSet
;
19 import org
.apache
.commons
.lang
.StringUtils
;
20 import org
.apache
.log4j
.Logger
;
21 import org
.springframework
.stereotype
.Component
;
23 import eu
.etaxonomy
.cdm
.api
.facade
.DerivedUnitFacade
;
24 import eu
.etaxonomy
.cdm
.api
.facade
.DerivedUnitFacade
.DerivedUnitType
;
25 import eu
.etaxonomy
.cdm
.api
.facade
.DerivedUnitFacadeNotSupportedException
;
26 import eu
.etaxonomy
.cdm
.io
.algaterra
.validation
.AlgaTerraTypeImportValidator
;
27 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.BerlinModelImportConfigurator
;
28 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.BerlinModelImportState
;
29 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.BerlinModelReferenceImport
;
30 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.BerlinModelTaxonNameImport
;
31 import eu
.etaxonomy
.cdm
.io
.common
.IOValidator
;
32 import eu
.etaxonomy
.cdm
.io
.common
.ResultSetPartitioner
;
33 import eu
.etaxonomy
.cdm
.model
.common
.CdmBase
;
34 import eu
.etaxonomy
.cdm
.model
.common
.IdentifiableSource
;
35 import eu
.etaxonomy
.cdm
.model
.name
.BotanicalName
;
36 import eu
.etaxonomy
.cdm
.model
.name
.Rank
;
37 import eu
.etaxonomy
.cdm
.model
.name
.SpecimenTypeDesignation
;
38 import eu
.etaxonomy
.cdm
.model
.name
.SpecimenTypeDesignationStatus
;
39 import eu
.etaxonomy
.cdm
.model
.name
.TaxonNameBase
;
40 import eu
.etaxonomy
.cdm
.model
.occurrence
.Collection
;
41 import eu
.etaxonomy
.cdm
.model
.occurrence
.DerivedUnit
;
42 import eu
.etaxonomy
.cdm
.model
.occurrence
.DerivedUnitBase
;
43 import eu
.etaxonomy
.cdm
.model
.occurrence
.FieldObservation
;
44 import eu
.etaxonomy
.cdm
.model
.reference
.Reference
;
53 public class AlgaTerraTypeImport
extends AlgaTerraSpecimenImportBase
{
54 private static final Logger logger
= Logger
.getLogger(AlgaTerraTypeImport
.class);
57 private static int modCount
= 5000;
58 private static final String pluralString
= "types";
59 private static final String dbTableName
= "TypeDesignation"; //??
61 protected String
getLocalityString(){
62 return "TypeLocality";
65 public AlgaTerraTypeImport(){
66 super(dbTableName
, pluralString
);
71 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery()
74 protected String
getIdQuery(BerlinModelImportState state
) {
75 String result
= " SELECT TypeDesignationId "
76 + " FROM TypeDesignation "
77 + " ORDER BY NameFk ";
82 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
85 protected String
getRecordQuery(BerlinModelImportConfigurator config
) {
88 " SELECT ts.*, ts.TypeSpecimenId as unitId, td.*, gz.ID as GazetteerId, gz.L2Code, gz.L3Code, gz.L4Code, gz.ISOCountry, gz.Country, ts.WaterBody, " +
89 " ts.RefFk as tsRefFk, ts.RefDetailFk as tsRefDetailFk, td.RefFk as tdRefFk, td.RefDetailFk as tdRefDetailFk " +
90 " FROM TypeSpecimenDesignation tsd "
91 + " LEFT OUTER JOIN TypeSpecimen AS ts ON tsd.TypeSpecimenFk = ts.TypeSpecimenId "
92 + " FULL OUTER JOIN TypeDesignation td ON td.TypeDesignationId = tsd.TypeDesignationFk "
93 + " LEFT OUTER JOIN TDWGGazetteer gz ON ts.TDWGGazetteerFk = gz.ID "
94 + " WHERE (td.TypeDesignationId IN (" + ID_LIST_TOKEN
+ ") )"
101 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
103 public boolean doPartition(ResultSetPartitioner partitioner
, BerlinModelImportState bmState
) {
104 boolean success
= true;
106 AlgaTerraImportState state
= (AlgaTerraImportState
)bmState
;
108 //TODO check that no duplicate vocabularies will be created, also remove redundant code here
109 // and in Specimen importer
111 makeVocabulariesAndFeatures(state
);
112 } catch (SQLException e1
) {
113 logger
.warn("Exception occurred when trying to create Type specimen vocabularies: " + e1
.getMessage());
114 e1
.printStackTrace();
119 Set
<TaxonNameBase
> namesToSave
= new HashSet
<TaxonNameBase
>();
121 Map
<String
, TaxonNameBase
> taxonNameMap
= (Map
<String
, TaxonNameBase
>) partitioner
.getObjectMap(BerlinModelTaxonNameImport
.NAMESPACE
);
122 Map
<String
, DerivedUnit
> ecoFactMap
= (Map
<String
, DerivedUnit
>) partitioner
.getObjectMap(AlgaTerraEcoFactImport
.ECO_FACT_FIELD_OBSERVATION_NAMESPACE
);
123 Map
<String
, DerivedUnit
> typeSpecimenMap
= (Map
<String
, DerivedUnit
>) partitioner
.getObjectMap(TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE
);
124 Map
<String
, Reference
> biblioRefMap
= (Map
<String
, Reference
>) partitioner
.getObjectMap(BerlinModelReferenceImport
.BIBLIO_REFERENCE_NAMESPACE
);
125 Map
<String
, Reference
> nomRefMap
= (Map
<String
, Reference
>) partitioner
.getObjectMap(BerlinModelReferenceImport
.NOM_REFERENCE_NAMESPACE
);
128 ResultSet rs
= partitioner
.getResultSet();
137 if ((i
++ % modCount
) == 0 && i
!= 1 ){ logger
.info("Type designations handled: " + (i
-1));}
139 int nameId
= rs
.getInt("nameFk");
140 int typeSpecimenId
= rs
.getInt("TypeSpecimenId");
141 int typeDesignationId
= rs
.getInt("TypeDesignationId");
142 Integer typeStatusFk
= nullSafeInt(rs
, "typeStatusFk");
143 Integer ecoFactId
= nullSafeInt(rs
, "ecoFactFk");
144 Integer tdRefFk
= nullSafeInt(rs
, "tdRefFk");
145 Integer tdRefDetailFk
= nullSafeInt(rs
, "tdRefDetailFk");
148 // String recordBasis = rs.getString("RecordBasis");
153 Reference
<?
> sourceRef
= state
.getTransactionalSourceReference();
156 //FIXME - depends on material category
157 // DerivedUnitType type = makeDerivedUnitType(recordBasis);
158 DerivedUnitType type
= DerivedUnitType
.Specimen
;
159 DerivedUnitFacade facade
= getDerivedUnit(state
, typeSpecimenId
, typeSpecimenMap
, type
, ecoFactMap
, ecoFactId
);
162 handleFieldObservationSpecimen(rs
, facade
, state
, partitioner
);
164 //TODO divide like in EcoFact (if necessary)
165 handleTypeSpecimenSpecificSpecimen(rs
,facade
, state
, biblioRefMap
, nomRefMap
, typeSpecimenId
);
167 handleFirstDerivedSpecimen(rs
, facade
, state
, partitioner
);
171 TaxonNameBase
<?
,?
> name
= getTaxonName(state
, taxonNameMap
, nameId
);
172 SpecimenTypeDesignation designation
= SpecimenTypeDesignation
.NewInstance();
173 SpecimenTypeDesignationStatus status
= getSpecimenTypeDesignationStatusByKey(typeStatusFk
);
174 designation
.setTypeSpecimen(facade
.innerDerivedUnit());
175 designation
.setTypeStatus(status
);
176 if (tdRefFk
!= null){
177 Reference
<?
> typeDesigRef
= getReferenceOnlyFromMaps(biblioRefMap
, nomRefMap
, String
.valueOf(tdRefFk
));
178 if (typeDesigRef
== null){
179 logger
.warn("Type designation reference not found in maps: " + tdRefFk
);
181 designation
.setCitation(typeDesigRef
);
185 if (tdRefDetailFk
!= null){
186 logger
.warn("TypeDesignation.RefDetailFk not yet implemented: " + typeDesignationId
);
190 name
.addTypeDesignation(designation
, true); //TODO check if true is correct
192 logger
.warn("Name could not be found for type designation " + typeDesignationId
);
194 namesToSave
.add(name
);
197 } catch (Exception e
) {
198 logger
.warn("Exception in TypeDesignation: TypeDesignationId " + typeDesignationId
+ ". " + e
.getMessage());
204 // logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
206 logger
.warn("Names to save: " + namesToSave
.size());
207 getNameService().save(namesToSave
);
210 } catch (SQLException e
) {
211 logger
.error("SQLException:" + e
);
217 protected String
getDerivedUnitNameSpace(){
218 return TYPE_SPECIMEN_DERIVED_UNIT_NAMESPACE
;
221 protected String
getFieldObservationNameSpace(){
222 return TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE
;
227 * @param taxonNameMap
231 private TaxonNameBase
<?
,?
> getTaxonName(AlgaTerraImportState state
, Map
<String
, TaxonNameBase
> taxonNameMap
, int nameId
) {
232 TaxonNameBase
<?
,?
> result
;
233 if (state
.getConfig().isDoTaxonNames()){
234 result
= taxonNameMap
.get(String
.valueOf(nameId
));
237 result
= BotanicalName
.NewInstance(Rank
.SPECIES());
242 private void handleTypeSpecimenSpecificSpecimen(ResultSet rs
, DerivedUnitFacade facade
, AlgaTerraImportState state
, Map
<String
, Reference
> biblioRefMap
, Map
<String
, Reference
> nomRefMap
, int typeSpecimenId
) throws SQLException
{
249 DerivedUnitBase
<?
> derivedUnit
= facade
.innerDerivedUnit();
252 String barcode
= rs
.getString("Barcode");
253 if (StringUtils
.isNotBlank(barcode
)){
254 facade
.setBarcode(barcode
);
257 //RefFk + RefDetailFk
258 Integer refFk
= nullSafeInt(rs
, "tsRefFk");
261 Reference
<?
> ref
= getReferenceOnlyFromMaps(biblioRefMap
, nomRefMap
, String
.valueOf(refFk
));
263 logger
.warn("TypeSpecimen reference (" + refFk
+ ")not found in biblioRef. TypeSpecimenId: " + typeSpecimenId
);
265 IdentifiableSource source
= IdentifiableSource
.NewInstance(ref
, null);
266 derivedUnit
.addSource(source
);
270 Integer refDetailFk
= nullSafeInt(rs
, "tsRefDetailFk");
271 if (refDetailFk
!= null){
272 logger
.warn("TypeSpecimen.RefDetailFk should always be NULL but wasn't: " + typeSpecimenId
);
282 * @param derivedUnitMap
288 private DerivedUnitFacade
getDerivedUnit(AlgaTerraImportState state
, int typeSpecimenId
, Map
<String
, DerivedUnit
> typeSpecimenMap
, DerivedUnitType type
, Map
<String
, DerivedUnit
> ecoFactMap
, Integer ecoFactId2
) {
289 //TODO implement ecoFact map - if not all null anymore
290 String typeKey
= String
.valueOf(typeSpecimenId
);
291 DerivedUnit derivedUnit
= typeSpecimenMap
.get(typeKey
);
292 DerivedUnitFacade facade
;
293 if (derivedUnit
== null){
294 facade
= DerivedUnitFacade
.NewInstance(type
);
295 typeSpecimenMap
.put(typeKey
, derivedUnit
);
298 facade
= DerivedUnitFacade
.NewInstance(derivedUnit
);
299 } catch (DerivedUnitFacadeNotSupportedException e
) {
300 logger
.error(e
.getMessage());
301 facade
= DerivedUnitFacade
.NewInstance(type
);
307 private DerivedUnitType
makeDerivedUnitType(String recordBasis
) {
308 DerivedUnitType result
= null;
309 if (StringUtils
.isBlank(recordBasis
)){
310 result
= DerivedUnitType
.DerivedUnit
;
311 } else if (recordBasis
.equalsIgnoreCase("FossileSpecimen")){
312 result
= DerivedUnitType
.Fossil
;
313 }else if (recordBasis
.equalsIgnoreCase("HumanObservation")){
314 result
= DerivedUnitType
.Observation
;
315 }else if (recordBasis
.equalsIgnoreCase("Literature")){
316 logger
.warn("Literature record basis not yet supported");
317 result
= DerivedUnitType
.DerivedUnit
;
318 }else if (recordBasis
.equalsIgnoreCase("LivingSpecimen")){
319 result
= DerivedUnitType
.LivingBeing
;
320 }else if (recordBasis
.equalsIgnoreCase("MachineObservation")){
321 logger
.warn("MachineObservation record basis not yet supported");
322 result
= DerivedUnitType
.Observation
;
323 }else if (recordBasis
.equalsIgnoreCase("PreservedSpecimen")){
324 result
= DerivedUnitType
.Specimen
;
330 private SpecimenTypeDesignationStatus
getSpecimenTypeDesignationStatusByKey(Integer typeStatusFk
) {
331 if (typeStatusFk
== null){ return null;
332 }else if (typeStatusFk
== 1) { return SpecimenTypeDesignationStatus
.HOLOTYPE();
333 }else if (typeStatusFk
== 2) { return SpecimenTypeDesignationStatus
.LECTOTYPE();
334 }else if (typeStatusFk
== 3) { return SpecimenTypeDesignationStatus
.NEOTYPE();
335 }else if (typeStatusFk
== 4) { return SpecimenTypeDesignationStatus
.EPITYPE();
336 }else if (typeStatusFk
== 5) { return SpecimenTypeDesignationStatus
.ISOLECTOTYPE();
337 }else if (typeStatusFk
== 6) { return SpecimenTypeDesignationStatus
.ISONEOTYPE();
338 }else if (typeStatusFk
== 7) { return SpecimenTypeDesignationStatus
.ISOTYPE();
339 }else if (typeStatusFk
== 8) { return SpecimenTypeDesignationStatus
.PARANEOTYPE();
340 }else if (typeStatusFk
== 9) { return SpecimenTypeDesignationStatus
.PARATYPE();
341 }else if (typeStatusFk
== 10) { return SpecimenTypeDesignationStatus
.SECOND_STEP_LECTOTYPE();
342 }else if (typeStatusFk
== 11) { return SpecimenTypeDesignationStatus
.SECOND_STEP_NEOTYPE();
343 }else if (typeStatusFk
== 12) { return SpecimenTypeDesignationStatus
.SYNTYPE();
344 }else if (typeStatusFk
== 13) { return SpecimenTypeDesignationStatus
.PARALECTOTYPE();
345 }else if (typeStatusFk
== 14) { return SpecimenTypeDesignationStatus
.ISOEPITYPE();
346 }else if (typeStatusFk
== 21) { return SpecimenTypeDesignationStatus
.ICONOTYPE();
347 }else if (typeStatusFk
== 22) { return SpecimenTypeDesignationStatus
.PHOTOTYPE();
348 }else if (typeStatusFk
== 30) { return SpecimenTypeDesignationStatus
.TYPE();
349 }else if (typeStatusFk
== 38) { return SpecimenTypeDesignationStatus
.ISOEPITYPE();
350 // }else if (typeStatusFk == 39) { return SpecimenTypeDesignationStatus.;
351 }else if (typeStatusFk
== 40) { return SpecimenTypeDesignationStatus
.ORIGINAL_MATERIAL();
353 logger
.warn("typeStatusFk undefined for " + typeStatusFk
);
354 return SpecimenTypeDesignationStatus
.TYPE();
361 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
363 public Map
<Object
, Map
<String
, ?
extends CdmBase
>> getRelatedObjectsForPartition(ResultSet rs
) {
367 Map
<Object
, Map
<String
, ?
extends CdmBase
>> result
= new HashMap
<Object
, Map
<String
, ?
extends CdmBase
>>();
370 Set
<String
> nameIdSet
= new HashSet
<String
>();
371 Set
<String
> ecoFieldObservationIdSet
= new HashSet
<String
>();
372 Set
<String
> typeSpecimenIdSet
= new HashSet
<String
>();
373 Set
<String
> termsIdSet
= new HashSet
<String
>();
374 Set
<String
> collectionIdSet
= new HashSet
<String
>();
375 Set
<String
> referenceIdSet
= new HashSet
<String
>();
378 handleForeignKey(rs
, nameIdSet
, "nameFk");
379 handleForeignKey(rs
, ecoFieldObservationIdSet
, "ecoFactFk");
380 handleForeignKey(rs
, typeSpecimenIdSet
, "TypeSpecimenId");
381 handleForeignKey(rs
, collectionIdSet
, "CollectionFk");
382 handleForeignKey(rs
, referenceIdSet
, "tsRefFk");
383 handleForeignKey(rs
, referenceIdSet
, "tdRefFk");
387 nameSpace
= BerlinModelTaxonNameImport
.NAMESPACE
;
388 cdmClass
= TaxonNameBase
.class;
390 Map
<String
, TaxonNameBase
> objectMap
= (Map
<String
, TaxonNameBase
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
391 result
.put(nameSpace
, objectMap
);
393 //eco fact field observation map
394 nameSpace
= AlgaTerraTypeImport
.ECO_FACT_FIELD_OBSERVATION_NAMESPACE
;
395 cdmClass
= FieldObservation
.class;
396 idSet
= ecoFieldObservationIdSet
;
397 Map
<String
, FieldObservation
> fieldObservationMap
= (Map
<String
, FieldObservation
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
398 result
.put(nameSpace
, fieldObservationMap
);
401 nameSpace
= AlgaTerraTypeImport
.TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE
;
402 cdmClass
= FieldObservation
.class;
403 idSet
= typeSpecimenIdSet
;
404 Map
<String
, FieldObservation
> typeSpecimenMap
= (Map
<String
, FieldObservation
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
405 result
.put(nameSpace
, typeSpecimenMap
);
409 nameSpace
= AlgaTerraCollectionImport
.NAMESPACE_COLLECTION
;
410 cdmClass
= Collection
.class;
411 idSet
= collectionIdSet
;
412 Map
<String
, Collection
> collectionMap
= (Map
<String
, Collection
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
413 result
.put(nameSpace
, collectionMap
);
416 nameSpace
= AlgaTerraCollectionImport
.NAMESPACE_SUBCOLLECTION
;
417 cdmClass
= Collection
.class;
418 idSet
= collectionIdSet
;
419 Map
<String
, Collection
> subCollectionMap
= (Map
<String
, Collection
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
420 result
.put(nameSpace
, subCollectionMap
);
423 nameSpace
= BerlinModelReferenceImport
.NOM_REFERENCE_NAMESPACE
;
424 cdmClass
= Reference
.class;
425 idSet
= referenceIdSet
;
426 Map
<String
, Reference
> nomReferenceMap
= (Map
<String
, Reference
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
427 result
.put(nameSpace
, nomReferenceMap
);
429 //biblio reference map
430 nameSpace
= BerlinModelReferenceImport
.BIBLIO_REFERENCE_NAMESPACE
;
431 cdmClass
= Reference
.class;
432 idSet
= referenceIdSet
;
433 Map
<String
, Reference
> biblioReferenceMap
= (Map
<String
, Reference
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
434 result
.put(nameSpace
, biblioReferenceMap
);
439 // nameSpace = AlgaTerraTypeImport.TERMS_NAMESPACE;
440 // cdmClass = FieldObservation.class;
441 // idSet = taxonIdSet;
442 // Map<String, DefinedTermBase> termMap = (Map<String, DefinedTermBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
443 // result.put(nameSpace, termMap);
448 } catch (SQLException e
) {
449 throw new RuntimeException(e
);
455 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
458 protected boolean doCheck(BerlinModelImportState state
){
459 IOValidator
<BerlinModelImportState
> validator
= new AlgaTerraTypeImportValidator();
460 return validator
.validate(state
);
465 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
467 protected boolean isIgnore(BerlinModelImportState state
){
468 return ! state
.getConfig().isDoTypes();