2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.algaterra
;
12 import java
.sql
.ResultSet
;
13 import java
.sql
.SQLException
;
14 import java
.util
.HashMap
;
15 import java
.util
.HashSet
;
19 import org
.apache
.commons
.lang
.StringUtils
;
20 import org
.apache
.log4j
.Logger
;
21 import org
.springframework
.stereotype
.Component
;
23 import eu
.etaxonomy
.cdm
.api
.facade
.DerivedUnitFacade
;
24 import eu
.etaxonomy
.cdm
.api
.facade
.DerivedUnitFacadeNotSupportedException
;
25 import eu
.etaxonomy
.cdm
.io
.algaterra
.validation
.AlgaTerraTypeImportValidator
;
26 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.BerlinModelImportConfigurator
;
27 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.BerlinModelImportState
;
28 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.BerlinModelReferenceImport
;
29 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.BerlinModelTaxonNameImport
;
30 import eu
.etaxonomy
.cdm
.io
.common
.IOValidator
;
31 import eu
.etaxonomy
.cdm
.io
.common
.ResultSetPartitioner
;
32 import eu
.etaxonomy
.cdm
.model
.common
.CdmBase
;
33 import eu
.etaxonomy
.cdm
.model
.common
.IdentifiableSource
;
34 import eu
.etaxonomy
.cdm
.model
.common
.OriginalSourceType
;
35 import eu
.etaxonomy
.cdm
.model
.name
.BotanicalName
;
36 import eu
.etaxonomy
.cdm
.model
.name
.Rank
;
37 import eu
.etaxonomy
.cdm
.model
.name
.SpecimenTypeDesignation
;
38 import eu
.etaxonomy
.cdm
.model
.name
.SpecimenTypeDesignationStatus
;
39 import eu
.etaxonomy
.cdm
.model
.name
.TaxonNameBase
;
40 import eu
.etaxonomy
.cdm
.model
.occurrence
.Collection
;
41 import eu
.etaxonomy
.cdm
.model
.occurrence
.DerivedUnit
;
42 import eu
.etaxonomy
.cdm
.model
.occurrence
.FieldUnit
;
43 import eu
.etaxonomy
.cdm
.model
.occurrence
.SpecimenOrObservationType
;
44 import eu
.etaxonomy
.cdm
.model
.reference
.Reference
;
53 public class AlgaTerraTypeImport
extends AlgaTerraSpecimenImportBase
{
54 private static final Logger logger
= Logger
.getLogger(AlgaTerraTypeImport
.class);
57 private static int modCount
= 5000;
58 private static final String pluralString
= "types";
59 private static final String dbTableName
= "TypeDesignation"; //??
61 protected String
getLocalityString(){
62 return "TypeLocality";
65 public AlgaTerraTypeImport(){
66 super(dbTableName
, pluralString
);
71 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery()
74 protected String
getIdQuery(BerlinModelImportState state
) {
75 String result
= " SELECT TypeDesignationId "
76 + " FROM TypeDesignation "
77 + " ORDER BY NameFk ";
82 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
85 protected String
getRecordQuery(BerlinModelImportConfigurator config
) {
88 " SELECT ts.*, ts.TypeSpecimenId as unitId, td.*, gz.ID as GazetteerId, gz.L2Code, gz.L3Code, gz.L4Code, gz.ISOCountry, gz.Country, ts.WaterBody, " +
89 " ts.RefFk as tsRefFk, ts.RefDetailFk as tsRefDetailFk, td.RefFk as tdRefFk, td.RefDetailFk as tdRefDetailFk " +
90 " FROM TypeSpecimenDesignation tsd "
91 + " LEFT OUTER JOIN TypeSpecimen AS ts ON tsd.TypeSpecimenFk = ts.TypeSpecimenId "
92 + " FULL OUTER JOIN TypeDesignation td ON td.TypeDesignationId = tsd.TypeDesignationFk "
93 + " LEFT OUTER JOIN TDWGGazetteer gz ON ts.TDWGGazetteerFk = gz.ID "
94 + " WHERE (td.TypeDesignationId IN (" + ID_LIST_TOKEN
+ ") )"
101 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
103 public boolean doPartition(ResultSetPartitioner partitioner
, BerlinModelImportState bmState
) {
104 boolean success
= true;
106 AlgaTerraImportState state
= (AlgaTerraImportState
)bmState
;
108 //TODO check that no duplicate vocabularies will be created, also remove redundant code here
109 // and in Specimen importer
111 makeVocabulariesAndFeatures(state
);
112 } catch (SQLException e1
) {
113 logger
.warn("Exception occurred when trying to create Type specimen vocabularies: " + e1
.getMessage());
114 e1
.printStackTrace();
119 Set
<TaxonNameBase
> namesToSave
= new HashSet
<TaxonNameBase
>();
121 Map
<String
, TaxonNameBase
> taxonNameMap
= (Map
<String
, TaxonNameBase
>) partitioner
.getObjectMap(BerlinModelTaxonNameImport
.NAMESPACE
);
122 Map
<String
, DerivedUnit
> ecoFactMap
= (Map
<String
, DerivedUnit
>) partitioner
.getObjectMap(AlgaTerraEcoFactImport
.ECO_FACT_FIELD_OBSERVATION_NAMESPACE
);
123 Map
<String
, DerivedUnit
> typeSpecimenMap
= (Map
<String
, DerivedUnit
>) partitioner
.getObjectMap(TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE
);
124 Map
<String
, Reference
> biblioRefMap
= (Map
<String
, Reference
>) partitioner
.getObjectMap(BerlinModelReferenceImport
.BIBLIO_REFERENCE_NAMESPACE
);
125 Map
<String
, Reference
> nomRefMap
= (Map
<String
, Reference
>) partitioner
.getObjectMap(BerlinModelReferenceImport
.NOM_REFERENCE_NAMESPACE
);
128 ResultSet rs
= partitioner
.getResultSet();
137 if ((i
++ % modCount
) == 0 && i
!= 1 ){ logger
.info("Type designations handled: " + (i
-1));}
139 int nameId
= rs
.getInt("nameFk");
140 int typeSpecimenId
= rs
.getInt("TypeSpecimenId");
141 int typeDesignationId
= rs
.getInt("TypeDesignationId");
142 Integer typeStatusFk
= nullSafeInt(rs
, "typeStatusFk");
143 Integer ecoFactId
= nullSafeInt(rs
, "ecoFactFk");
144 Integer tdRefFk
= nullSafeInt(rs
, "tdRefFk");
145 Integer tdRefDetailFk
= nullSafeInt(rs
, "tdRefDetailFk");
148 // String recordBasis = rs.getString("RecordBasis");
153 Reference
<?
> sourceRef
= state
.getTransactionalSourceReference();
156 //FIXME - depends on material category
157 // DerivedUnitType type = makeDerivedUnitType(recordBasis);
158 if (i
<=2){logger
.warn("RecordBasis not yet implemented in TypeImport");}
160 SpecimenOrObservationType type
= SpecimenOrObservationType
.PreservedSpecimen
;
161 DerivedUnitFacade facade
= getDerivedUnit(state
, typeSpecimenId
, typeSpecimenMap
, type
, ecoFactMap
, ecoFactId
);
164 handleFieldObservationSpecimen(rs
, facade
, state
, partitioner
);
166 //TODO divide like in EcoFact (if necessary)
167 handleTypeSpecimenSpecificSpecimen(rs
,facade
, state
, biblioRefMap
, nomRefMap
, typeSpecimenId
);
169 handleFirstDerivedSpecimen(rs
, facade
, state
, partitioner
);
173 TaxonNameBase
<?
,?
> name
= getTaxonName(state
, taxonNameMap
, nameId
);
174 SpecimenTypeDesignation designation
= SpecimenTypeDesignation
.NewInstance();
175 SpecimenTypeDesignationStatus status
= getSpecimenTypeDesignationStatusByKey(typeStatusFk
);
176 designation
.setTypeSpecimen(facade
.innerDerivedUnit());
177 designation
.setTypeStatus(status
);
178 if (tdRefFk
!= null){
179 Reference
<?
> typeDesigRef
= getReferenceOnlyFromMaps(biblioRefMap
, nomRefMap
, String
.valueOf(tdRefFk
));
180 if (typeDesigRef
== null){
181 logger
.warn("Type designation reference not found in maps: " + tdRefFk
);
183 designation
.setCitation(typeDesigRef
);
187 if (tdRefDetailFk
!= null){
188 logger
.warn("TypeDesignation.RefDetailFk not yet implemented: " + typeDesignationId
);
192 name
.addTypeDesignation(designation
, true); //TODO check if true is correct
194 logger
.warn("Name could not be found for type designation " + typeDesignationId
);
196 namesToSave
.add(name
);
199 } catch (Exception e
) {
200 logger
.warn("Exception in TypeDesignation: TypeDesignationId " + typeDesignationId
+ ". " + e
.getMessage());
206 // logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
208 logger
.warn("Names to save: " + namesToSave
.size());
209 getNameService().save(namesToSave
);
212 } catch (SQLException e
) {
213 logger
.error("SQLException:" + e
);
219 protected String
getDerivedUnitNameSpace(){
220 return TYPE_SPECIMEN_DERIVED_UNIT_NAMESPACE
;
223 protected String
getFieldObservationNameSpace(){
224 return TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE
;
229 * @param taxonNameMap
233 private TaxonNameBase
<?
,?
> getTaxonName(AlgaTerraImportState state
, Map
<String
, TaxonNameBase
> taxonNameMap
, int nameId
) {
234 TaxonNameBase
<?
,?
> result
;
235 if (state
.getConfig().isDoTaxonNames()){
236 result
= taxonNameMap
.get(String
.valueOf(nameId
));
239 result
= BotanicalName
.NewInstance(Rank
.SPECIES());
244 private void handleTypeSpecimenSpecificSpecimen(ResultSet rs
, DerivedUnitFacade facade
, AlgaTerraImportState state
, Map
<String
, Reference
> biblioRefMap
, Map
<String
, Reference
> nomRefMap
, int typeSpecimenId
) throws SQLException
{
251 DerivedUnit derivedUnit
= facade
.innerDerivedUnit();
254 String barcode
= rs
.getString("Barcode");
255 if (StringUtils
.isNotBlank(barcode
)){
256 facade
.setBarcode(barcode
);
259 //RefFk + RefDetailFk
260 Integer refFk
= nullSafeInt(rs
, "tsRefFk");
263 Reference
<?
> ref
= getReferenceOnlyFromMaps(biblioRefMap
, nomRefMap
, String
.valueOf(refFk
));
265 logger
.warn("TypeSpecimen reference (" + refFk
+ ")not found in biblioRef. TypeSpecimenId: " + typeSpecimenId
);
267 IdentifiableSource source
= IdentifiableSource
.NewPrimarySourceInstance(ref
, null);
268 derivedUnit
.addSource(source
);
272 Integer refDetailFk
= nullSafeInt(rs
, "tsRefDetailFk");
273 if (refDetailFk
!= null){
274 logger
.warn("TypeSpecimen.RefDetailFk should always be NULL but wasn't: " + typeSpecimenId
);
284 * @param derivedUnitMap
290 private DerivedUnitFacade
getDerivedUnit(AlgaTerraImportState state
, int typeSpecimenId
, Map
<String
, DerivedUnit
> typeSpecimenMap
, SpecimenOrObservationType type
, Map
<String
, DerivedUnit
> ecoFactMap
, Integer ecoFactId2
) {
291 //TODO implement ecoFact map - if not all null anymore
292 String typeKey
= String
.valueOf(typeSpecimenId
);
293 DerivedUnit derivedUnit
= typeSpecimenMap
.get(typeKey
);
294 DerivedUnitFacade facade
;
295 if (derivedUnit
== null){
296 facade
= DerivedUnitFacade
.NewInstance(type
);
297 typeSpecimenMap
.put(typeKey
, derivedUnit
);
300 facade
= DerivedUnitFacade
.NewInstance(derivedUnit
);
301 } catch (DerivedUnitFacadeNotSupportedException e
) {
302 logger
.error(e
.getMessage());
303 facade
= DerivedUnitFacade
.NewInstance(type
);
309 // private DerivedUnitType makeDerivedUnitType(String recordBasis) {
310 // DerivedUnitType result = null;
311 // if (StringUtils.isBlank(recordBasis)){
312 // result = DerivedUnitType.DerivedUnit;
313 // } else if (recordBasis.equalsIgnoreCase("FossileSpecimen")){
314 // result = DerivedUnitType.Fossil;
315 // }else if (recordBasis.equalsIgnoreCase("HumanObservation")){
316 // result = DerivedUnitType.Observation;
317 // }else if (recordBasis.equalsIgnoreCase("Literature")){
318 // logger.warn("Literature record basis not yet supported");
319 // result = DerivedUnitType.DerivedUnit;
320 // }else if (recordBasis.equalsIgnoreCase("LivingSpecimen")){
321 // result = DerivedUnitType.LivingBeing;
322 // }else if (recordBasis.equalsIgnoreCase("MachineObservation")){
323 // logger.warn("MachineObservation record basis not yet supported");
324 // result = DerivedUnitType.Observation;
325 // }else if (recordBasis.equalsIgnoreCase("PreservedSpecimen")){
326 // result = DerivedUnitType.Specimen;
332 private SpecimenTypeDesignationStatus
getSpecimenTypeDesignationStatusByKey(Integer typeStatusFk
) {
333 if (typeStatusFk
== null){ return null;
334 }else if (typeStatusFk
== 1) { return SpecimenTypeDesignationStatus
.HOLOTYPE();
335 }else if (typeStatusFk
== 2) { return SpecimenTypeDesignationStatus
.LECTOTYPE();
336 }else if (typeStatusFk
== 3) { return SpecimenTypeDesignationStatus
.NEOTYPE();
337 }else if (typeStatusFk
== 4) { return SpecimenTypeDesignationStatus
.EPITYPE();
338 }else if (typeStatusFk
== 5) { return SpecimenTypeDesignationStatus
.ISOLECTOTYPE();
339 }else if (typeStatusFk
== 6) { return SpecimenTypeDesignationStatus
.ISONEOTYPE();
340 }else if (typeStatusFk
== 7) { return SpecimenTypeDesignationStatus
.ISOTYPE();
341 }else if (typeStatusFk
== 8) { return SpecimenTypeDesignationStatus
.PARANEOTYPE();
342 }else if (typeStatusFk
== 9) { return SpecimenTypeDesignationStatus
.PARATYPE();
343 }else if (typeStatusFk
== 10) { return SpecimenTypeDesignationStatus
.SECOND_STEP_LECTOTYPE();
344 }else if (typeStatusFk
== 11) { return SpecimenTypeDesignationStatus
.SECOND_STEP_NEOTYPE();
345 }else if (typeStatusFk
== 12) { return SpecimenTypeDesignationStatus
.SYNTYPE();
346 }else if (typeStatusFk
== 13) { return SpecimenTypeDesignationStatus
.PARALECTOTYPE();
347 }else if (typeStatusFk
== 14) { return SpecimenTypeDesignationStatus
.ISOEPITYPE();
348 }else if (typeStatusFk
== 21) { return SpecimenTypeDesignationStatus
.ICONOTYPE();
349 }else if (typeStatusFk
== 22) { return SpecimenTypeDesignationStatus
.PHOTOTYPE();
350 }else if (typeStatusFk
== 30) { return SpecimenTypeDesignationStatus
.TYPE();
351 }else if (typeStatusFk
== 38) { return SpecimenTypeDesignationStatus
.ISOEPITYPE();
352 // }else if (typeStatusFk == 39) { return SpecimenTypeDesignationStatus.;
353 }else if (typeStatusFk
== 40) { return SpecimenTypeDesignationStatus
.ORIGINAL_MATERIAL();
355 logger
.warn("typeStatusFk undefined for " + typeStatusFk
);
356 return SpecimenTypeDesignationStatus
.TYPE();
363 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
365 public Map
<Object
, Map
<String
, ?
extends CdmBase
>> getRelatedObjectsForPartition(ResultSet rs
) {
369 Map
<Object
, Map
<String
, ?
extends CdmBase
>> result
= new HashMap
<Object
, Map
<String
, ?
extends CdmBase
>>();
372 Set
<String
> nameIdSet
= new HashSet
<String
>();
373 Set
<String
> ecoFieldObservationIdSet
= new HashSet
<String
>();
374 Set
<String
> typeSpecimenIdSet
= new HashSet
<String
>();
375 Set
<String
> termsIdSet
= new HashSet
<String
>();
376 Set
<String
> collectionIdSet
= new HashSet
<String
>();
377 Set
<String
> referenceIdSet
= new HashSet
<String
>();
380 handleForeignKey(rs
, nameIdSet
, "nameFk");
381 handleForeignKey(rs
, ecoFieldObservationIdSet
, "ecoFactFk");
382 handleForeignKey(rs
, typeSpecimenIdSet
, "TypeSpecimenId");
383 handleForeignKey(rs
, collectionIdSet
, "CollectionFk");
384 handleForeignKey(rs
, referenceIdSet
, "tsRefFk");
385 handleForeignKey(rs
, referenceIdSet
, "tdRefFk");
389 nameSpace
= BerlinModelTaxonNameImport
.NAMESPACE
;
390 cdmClass
= TaxonNameBase
.class;
392 Map
<String
, TaxonNameBase
> objectMap
= (Map
<String
, TaxonNameBase
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
393 result
.put(nameSpace
, objectMap
);
395 //eco fact field observation map
396 nameSpace
= AlgaTerraTypeImport
.ECO_FACT_FIELD_OBSERVATION_NAMESPACE
;
397 cdmClass
= FieldUnit
.class;
398 idSet
= ecoFieldObservationIdSet
;
399 Map
<String
, FieldUnit
> fieldObservationMap
= (Map
<String
, FieldUnit
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
400 result
.put(nameSpace
, fieldObservationMap
);
403 nameSpace
= AlgaTerraTypeImport
.TYPE_SPECIMEN_FIELD_OBSERVATION_NAMESPACE
;
404 cdmClass
= FieldUnit
.class;
405 idSet
= typeSpecimenIdSet
;
406 Map
<String
, FieldUnit
> typeSpecimenMap
= (Map
<String
, FieldUnit
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
407 result
.put(nameSpace
, typeSpecimenMap
);
411 nameSpace
= AlgaTerraCollectionImport
.NAMESPACE_COLLECTION
;
412 cdmClass
= Collection
.class;
413 idSet
= collectionIdSet
;
414 Map
<String
, Collection
> collectionMap
= (Map
<String
, Collection
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
415 result
.put(nameSpace
, collectionMap
);
418 nameSpace
= AlgaTerraCollectionImport
.NAMESPACE_SUBCOLLECTION
;
419 cdmClass
= Collection
.class;
420 idSet
= collectionIdSet
;
421 Map
<String
, Collection
> subCollectionMap
= (Map
<String
, Collection
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
422 result
.put(nameSpace
, subCollectionMap
);
425 nameSpace
= BerlinModelReferenceImport
.NOM_REFERENCE_NAMESPACE
;
426 cdmClass
= Reference
.class;
427 idSet
= referenceIdSet
;
428 Map
<String
, Reference
> nomReferenceMap
= (Map
<String
, Reference
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
429 result
.put(nameSpace
, nomReferenceMap
);
431 //biblio reference map
432 nameSpace
= BerlinModelReferenceImport
.BIBLIO_REFERENCE_NAMESPACE
;
433 cdmClass
= Reference
.class;
434 idSet
= referenceIdSet
;
435 Map
<String
, Reference
> biblioReferenceMap
= (Map
<String
, Reference
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
436 result
.put(nameSpace
, biblioReferenceMap
);
441 // nameSpace = AlgaTerraTypeImport.TERMS_NAMESPACE;
442 // cdmClass = FieldObservation.class;
443 // idSet = taxonIdSet;
444 // Map<String, DefinedTermBase> termMap = (Map<String, DefinedTermBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
445 // result.put(nameSpace, termMap);
450 } catch (SQLException e
) {
451 throw new RuntimeException(e
);
457 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
460 protected boolean doCheck(BerlinModelImportState state
){
461 IOValidator
<BerlinModelImportState
> validator
= new AlgaTerraTypeImportValidator();
462 return validator
.validate(state
);
467 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
469 protected boolean isIgnore(BerlinModelImportState state
){
470 return ! state
.getConfig().isDoTypes();