26 |
26 |
|
27 |
27 |
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
|
28 |
28 |
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
|
|
29 |
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeNotSupportedException;
|
29 |
30 |
import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraSpecimenImportValidator;
|
30 |
31 |
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase;
|
31 |
32 |
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
|
... | ... | |
41 |
42 |
import eu.etaxonomy.cdm.model.common.Marker;
|
42 |
43 |
import eu.etaxonomy.cdm.model.common.MarkerType;
|
43 |
44 |
import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
|
|
45 |
import eu.etaxonomy.cdm.model.common.TermVocabulary;
|
44 |
46 |
import eu.etaxonomy.cdm.model.common.TimePeriod;
|
45 |
47 |
import eu.etaxonomy.cdm.model.description.CategoricalData;
|
46 |
48 |
import eu.etaxonomy.cdm.model.description.DescriptionBase;
|
47 |
49 |
import eu.etaxonomy.cdm.model.description.Feature;
|
48 |
50 |
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
|
|
51 |
import eu.etaxonomy.cdm.model.description.MeasurementUnit;
|
|
52 |
import eu.etaxonomy.cdm.model.description.Modifier;
|
|
53 |
import eu.etaxonomy.cdm.model.description.QuantitativeData;
|
49 |
54 |
import eu.etaxonomy.cdm.model.description.SpecimenDescription;
|
50 |
55 |
import eu.etaxonomy.cdm.model.description.State;
|
|
56 |
import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
|
|
57 |
import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
|
51 |
58 |
import eu.etaxonomy.cdm.model.description.TaxonDescription;
|
52 |
59 |
import eu.etaxonomy.cdm.model.description.TextData;
|
53 |
60 |
import eu.etaxonomy.cdm.model.location.NamedArea;
|
... | ... | |
57 |
64 |
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
|
58 |
65 |
import eu.etaxonomy.cdm.model.name.BotanicalName;
|
59 |
66 |
import eu.etaxonomy.cdm.model.name.Rank;
|
|
67 |
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
|
60 |
68 |
import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
|
61 |
69 |
import eu.etaxonomy.cdm.model.reference.Reference;
|
62 |
70 |
import eu.etaxonomy.cdm.model.taxon.Taxon;
|
... | ... | |
72 |
80 |
public class AlgaTerraSpecimenImport extends BerlinModelImportBase {
|
73 |
81 |
private static final Logger logger = Logger.getLogger(AlgaTerraSpecimenImport.class);
|
74 |
82 |
|
75 |
|
public static final String FIELD_OBSERVATION_NAMESPACE = "FieldObservation";
|
|
83 |
public static final String ECO_FACT_NAMESPACE = "EcoFact";
|
76 |
84 |
public static final String TERMS_NAMESPACE = "ALGA_TERRA_TERMS";
|
77 |
85 |
|
78 |
|
//move to transformrer
|
|
86 |
//TODO move to transformrer
|
79 |
87 |
final static UUID uuidMarkerAlkalinity = UUID.fromString("e52d0ea2-0c1f-4d95-ae6d-e21ab317c594");
|
80 |
88 |
final static UUID uuidRefSystemGps = UUID.fromString("c23e4928-c137-4e4a-b6ab-b430da3d0b94");
|
81 |
89 |
final static UUID uuidFeatureSpecimenCommunity = UUID.fromString("3ff5b1ab-3999-4b5a-b8f7-01fd2f6c12c7");
|
82 |
90 |
final static UUID uuidFeatureAdditionalData = UUID.fromString("0ac82ab8-2c2b-4953-98eb-a9f718eb9c57");
|
|
91 |
final static UUID uuidFeatureHabitatExplanation = UUID.fromString("6fe32295-61a3-44fc-9fcf-a85790ea888f");
|
83 |
92 |
|
84 |
93 |
final static UUID uuidVocAlgaTerraClimate = UUID.fromString("b0a677c6-8bb6-43f4-b1b8-fc377a10feb5");
|
85 |
94 |
final static UUID uuidVocAlgaTerraHabitat = UUID.fromString("06f30114-e19c-4e7d-a8e5-5488c41fcbc5");
|
... | ... | |
89 |
98 |
final static UUID uuidFeatureAlgaTerraHabitat = UUID.fromString("7def3fc2-cdc5-4739-8e13-62edbd053415");
|
90 |
99 |
final static UUID uuidFeatureAlgaTerraLifeForm = UUID.fromString("9b657901-1b0d-4a2a-8d21-dd8c1413e2e6");
|
91 |
100 |
|
|
101 |
final static UUID uuidVocParameter = UUID.fromString("45888b40-5bbb-4293-aa1e-02479796cd7c");
|
|
102 |
final static UUID uuidStatMeasureSingleValue = UUID.fromString("eb4c3d98-4d4b-4c37-8eb4-17315ce79920");
|
|
103 |
final static UUID uuidMeasurementValueModifier = UUID.fromString("0218a7a3-f6c0-4d06-a4f8-6b50b73aef5e");
|
92 |
104 |
|
|
105 |
final static UUID uuidModifierLowerThan = UUID.fromString("2b500085-6bef-4003-b6ea-e0ad0237d79d");
|
|
106 |
final static UUID uuidModifierGreaterThan = UUID.fromString("828df49d-c745-48f7-b083-0ada43356c34");
|
93 |
107 |
|
94 |
108 |
private static int modCount = 5000;
|
95 |
109 |
private static final String pluralString = "specimen and observation";
|
... | ... | |
154 |
168 |
Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
|
155 |
169 |
|
156 |
170 |
Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
|
157 |
|
Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>) partitioner.getObjectMap(FIELD_OBSERVATION_NAMESPACE);
|
|
171 |
Map<String, DerivedUnit> ecoFactMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(ECO_FACT_NAMESPACE);
|
158 |
172 |
|
159 |
173 |
ResultSet rs = partitioner.getResultSet();
|
160 |
174 |
|
... | ... | |
178 |
192 |
Reference<?> sourceRef = state.getTransactionalSourceReference();
|
179 |
193 |
|
180 |
194 |
//facade
|
181 |
|
FieldObservation fieldObservation = getFieldObservation(ecoFactId, fieldObservationMap);
|
182 |
195 |
DerivedUnitType type = makeDerivedUnitType(recordBasis);
|
183 |
|
DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(type, fieldObservation);
|
184 |
|
|
|
196 |
DerivedUnitFacade facade = getDerivedUnit(state, ecoFactId, ecoFactMap, type);
|
|
197 |
|
185 |
198 |
//field observation
|
186 |
199 |
handleSingleSpecimen(rs, facade, state);
|
187 |
200 |
|
|
201 |
state.setCurrentFieldObservationNotNew(false);
|
|
202 |
|
188 |
203 |
//description element
|
189 |
204 |
TaxonDescription taxonDescription = getTaxonDescription(state, newTaxonId, taxonMap, factId, sourceRef);
|
190 |
205 |
IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
|
... | ... | |
295 |
310 |
|
296 |
311 |
|
297 |
312 |
private void handleSingleSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state) throws SQLException {
|
298 |
|
//CollectionFk, Collector, AltitudeMethod,
|
299 |
|
//ISOCountrySub, CreatedWhen/Who/Updated/who
|
300 |
|
|
301 |
|
//P1-10Value/Unit/Parameter/Method
|
302 |
|
|
|
313 |
//FIXME missing fields #3084, #3085, #3080
|
303 |
314 |
try {
|
304 |
315 |
Object alkalinityFlag = rs.getBoolean("AlkalinityFlag");
|
305 |
316 |
|
... | ... | |
329 |
340 |
|
330 |
341 |
|
331 |
342 |
|
332 |
|
FieldObservation fieldObservation = facade.innerFieldObservation();
|
|
343 |
FieldObservation fieldObservation = facade.getFieldObservation(true);
|
333 |
344 |
|
334 |
345 |
//alkalinity marker
|
335 |
346 |
if (alkalinityFlag != null){
|
... | ... | |
376 |
387 |
|
377 |
388 |
//habitat, ecology, community, etc.
|
378 |
389 |
DescriptionBase<?> fieldDescription = getFieldObservationDescription(facade);
|
|
390 |
|
379 |
391 |
addCategoricalValue(state, fieldDescription, climateUuid, uuidFeatureAlgaTerraClimate);
|
380 |
392 |
addCategoricalValue(state, fieldDescription, habitatUuid, Feature.HABITAT().getUuid());
|
381 |
393 |
addCategoricalValue(state, fieldDescription, lifeFormUuid, uuidFeatureAlgaTerraLifeForm);
|
382 |
394 |
|
383 |
395 |
if (isNotBlank(habitat)){
|
384 |
|
//FIXME
|
385 |
|
facade.setEcology(habitat); //or use an own feature ??
|
|
396 |
Feature habitatExplanation = getFeature(state, uuidFeatureHabitatExplanation, "Habitat Explanation", "HabitatExplanation", null, null);
|
|
397 |
TextData textData = TextData.NewInstance(habitatExplanation);
|
|
398 |
textData.putText(Language.DEFAULT(), habitat);
|
|
399 |
getFieldObservationDescription(facade).addElement(textData);
|
386 |
400 |
}
|
387 |
401 |
if (isNotBlank(community)){
|
388 |
402 |
Feature communityFeature = getFeature(state, uuidFeatureSpecimenCommunity, "Community", "The community of a specimen (e.g. other algae in the same sample)", null, null);
|
... | ... | |
407 |
421 |
makeAreas(state, rs, facade);
|
408 |
422 |
|
409 |
423 |
//parameters
|
410 |
|
//TODO
|
|
424 |
makeParameter(state, rs, getFieldObservationDescription(facade));
|
411 |
425 |
|
412 |
426 |
//collection
|
413 |
427 |
String voucher = rs.getString("Voucher");
|
... | ... | |
422 |
436 |
//TODO id, created for fact + ecoFact
|
423 |
437 |
// this.doIdCreatedUpdatedNotes(state, descriptionElement, rs, id, namespace);
|
424 |
438 |
|
425 |
|
} catch (IllegalArgumentException e) {
|
426 |
|
throw e;
|
|
439 |
} catch (Exception e) {
|
|
440 |
throw new RuntimeException(e);
|
427 |
441 |
}
|
428 |
442 |
|
429 |
443 |
}
|
430 |
444 |
|
431 |
445 |
|
|
446 |
private void makeParameter(AlgaTerraImportState state, ResultSet rs, DescriptionBase<?> descriptionBase) throws SQLException {
|
|
447 |
for (int i = 1; i <= 10; i++){
|
|
448 |
String valueStr = rs.getString(String.format("P%dValue", i));
|
|
449 |
String unitStr = rs.getString(String.format("P%dUnit", i));
|
|
450 |
String parameter = rs.getString(String.format("P%dParameter", i));
|
|
451 |
String method = rs.getString(String.format("P%dMethod", i));
|
|
452 |
|
|
453 |
//method
|
|
454 |
if (StringUtils.isNotBlank(method)){
|
|
455 |
logger.warn("Methods not yet handled: " + method);
|
|
456 |
}
|
|
457 |
//parameter
|
|
458 |
TermVocabulary<Feature> vocParameter = getVocabulary(uuidVocParameter, "Feature vocabulary for AlgaTerra measurement parameters", "Parameters", null, null, false, Feature.COMMON_NAME());
|
|
459 |
if (StringUtils.isNotBlank(parameter)){
|
|
460 |
UUID featureUuid = getParameterFeatureUuid(state, parameter);
|
|
461 |
Feature feature = getFeature(state, featureUuid, parameter, parameter, null, vocParameter);
|
|
462 |
QuantitativeData quantData = QuantitativeData.NewInstance(feature);
|
|
463 |
|
|
464 |
//unit
|
|
465 |
MeasurementUnit unit = getMeasurementUnit(state, unitStr);
|
|
466 |
quantData.setUnit(unit);
|
|
467 |
try {
|
|
468 |
|
|
469 |
Set<Modifier> valueModifier = new HashSet<Modifier>();
|
|
470 |
valueStr = normalizeAndModifyValue(state, valueStr, valueModifier);
|
|
471 |
//value
|
|
472 |
Float valueFlt = Float.valueOf(valueStr); //TODO maybe change model to Double ??
|
|
473 |
|
|
474 |
StatisticalMeasure measureSingleValue = getStatisticalMeasure(state, uuidStatMeasureSingleValue, "Value", "Single measurement value", null, null);
|
|
475 |
StatisticalMeasurementValue value = StatisticalMeasurementValue.NewInstance(measureSingleValue, valueFlt);
|
|
476 |
quantData.addStatisticalValue(value);
|
|
477 |
descriptionBase.addElement(quantData);
|
|
478 |
|
|
479 |
} catch (NumberFormatException e) {
|
|
480 |
logger.warn(String.format("Value '%s' can't be converted to double. Parameter %s not imported.", valueStr, parameter));
|
|
481 |
}
|
|
482 |
}else if (isNotBlank(valueStr) || isNotBlank(unitStr) ){
|
|
483 |
logger.warn("There is value or unit without parameter: " + i);
|
|
484 |
}
|
|
485 |
|
|
486 |
|
|
487 |
}
|
|
488 |
|
|
489 |
}
|
|
490 |
|
|
491 |
private String normalizeAndModifyValue(AlgaTerraImportState state, String valueStr, Set<Modifier> valueModifier) {
|
|
492 |
valueStr = valueStr.replace(",", ".");
|
|
493 |
if (valueStr.startsWith("<")){
|
|
494 |
TermVocabulary<Modifier> measurementValueModifierVocabulary = getVocabulary(uuidMeasurementValueModifier, "Measurement value modifier", "Measurement value modifier", null, null, false, Modifier.NewInstance());
|
|
495 |
Modifier modifier = getModifier(state, uuidModifierLowerThan, "Lower", "Lower than the given measurement value", "<", measurementValueModifierVocabulary);
|
|
496 |
valueModifier.add(modifier);
|
|
497 |
valueStr = valueStr.replace("<", "");
|
|
498 |
}
|
|
499 |
if (valueStr.startsWith(">")){
|
|
500 |
TermVocabulary<Modifier> measurementValueModifierVocabulary = getVocabulary(uuidMeasurementValueModifier, "Measurement value modifier", "Measurement value modifier", null, null, false, Modifier.NewInstance());
|
|
501 |
Modifier modifier = getModifier(state, uuidModifierGreaterThan, "Lower", "Lower than the given measurement value", "<", measurementValueModifierVocabulary);
|
|
502 |
valueModifier.add(modifier);
|
|
503 |
valueStr = valueStr.replace(">", "");
|
|
504 |
}
|
|
505 |
return valueStr;
|
|
506 |
}
|
|
507 |
|
|
508 |
|
|
509 |
|
|
510 |
private UUID getParameterFeatureUuid(AlgaTerraImportState state, String key) {
|
|
511 |
//TODO define some UUIDs in Transformer
|
|
512 |
UUID uuid = state.getParameterFeatureUuid(key);
|
|
513 |
if (uuid == null){
|
|
514 |
uuid = UUID.randomUUID();
|
|
515 |
state.putParameterFeatureUuid(key, uuid);
|
|
516 |
}
|
|
517 |
return uuid;
|
|
518 |
}
|
|
519 |
|
|
520 |
|
|
521 |
|
|
522 |
/**
|
|
523 |
* TODO move to InputTransformerBase
|
|
524 |
* @param state
|
|
525 |
* @param unitStr
|
|
526 |
* @return
|
|
527 |
*/
|
|
528 |
private MeasurementUnit getMeasurementUnit(AlgaTerraImportState state, String unitStr) {
|
|
529 |
MeasurementUnit result = null;
|
|
530 |
if (StringUtils.isNotBlank(unitStr)){
|
|
531 |
UUID uuidMeasurementUnitMgL = UUID.fromString("7ac302c5-3cbd-4334-964a-bf5d11eb9ead");
|
|
532 |
UUID uuidMeasurementUnitMolMol = UUID.fromString("96b78d78-3e49-448f-8100-e7779b71dd53");
|
|
533 |
UUID uuidMeasurementUnitMicroMolSiL = UUID.fromString("2cb8bc85-a4af-42f1-b80b-34c36c9f75d4");
|
|
534 |
UUID uuidMeasurementUnitMicroMolL = UUID.fromString("a631f62e-377e-405c-bd1a-76885b13a72b");
|
|
535 |
UUID uuidMeasurementUnitDegreeC = UUID.fromString("55222aec-d5be-413e-8db7-d9a48c316c6c");
|
|
536 |
UUID uuidMeasurementUnitPercent = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
|
|
537 |
UUID uuidMeasurementUnitCm = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
|
|
538 |
UUID uuidMeasurementUnitMicroSiCm = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
|
|
539 |
|
|
540 |
|
|
541 |
if (unitStr.equalsIgnoreCase("mg/L")){
|
|
542 |
return getMeasurementUnit(state, uuidMeasurementUnitMgL, unitStr, unitStr, unitStr, null);
|
|
543 |
}else if (unitStr.equalsIgnoreCase("mol/mol")){
|
|
544 |
return result = getMeasurementUnit(state, uuidMeasurementUnitMolMol, unitStr, unitStr, unitStr, null);
|
|
545 |
}else if (unitStr.equalsIgnoreCase("\u00B5mol Si/L")){ //µmol Si/L
|
|
546 |
return getMeasurementUnit(state, uuidMeasurementUnitMicroMolSiL, unitStr, unitStr, unitStr, null);
|
|
547 |
}else if (unitStr.equalsIgnoreCase("\u00B5mol/L")){ //µmol/L
|
|
548 |
return getMeasurementUnit(state, uuidMeasurementUnitMicroMolL, unitStr, unitStr, unitStr, null);
|
|
549 |
}else if (unitStr.equalsIgnoreCase("\u00B0C")){ //°C
|
|
550 |
return getMeasurementUnit(state, uuidMeasurementUnitDegreeC, unitStr, unitStr, unitStr, null);
|
|
551 |
}else if (unitStr.equalsIgnoreCase("%")){
|
|
552 |
return getMeasurementUnit(state, uuidMeasurementUnitPercent, unitStr, unitStr, unitStr, null);
|
|
553 |
}else if (unitStr.equalsIgnoreCase("cm")){
|
|
554 |
return getMeasurementUnit(state, uuidMeasurementUnitCm, unitStr, unitStr, unitStr, null);
|
|
555 |
}else if (unitStr.equalsIgnoreCase("\u00B5S/cm")){ //µS/cm
|
|
556 |
return getMeasurementUnit(state, uuidMeasurementUnitMicroSiCm, unitStr, unitStr, unitStr, null);
|
|
557 |
}else{
|
|
558 |
logger.warn("MeasurementUnit was not recognized");
|
|
559 |
return null;
|
|
560 |
}
|
|
561 |
}else{
|
|
562 |
return null;
|
|
563 |
}
|
|
564 |
}
|
|
565 |
|
|
566 |
|
|
567 |
|
432 |
568 |
private void addCategoricalValue(AlgaTerraImportState importState, DescriptionBase description, String uuidTerm, UUID featureUuid) {
|
433 |
569 |
if (uuidTerm != null){
|
434 |
570 |
State state = this.getStateTerm(importState, UUID.fromString(uuidTerm));
|
... | ... | |
539 |
675 |
}
|
540 |
676 |
}
|
541 |
677 |
|
542 |
|
private FieldObservation getFieldObservation(int ecoFactId, Map<String, FieldObservation> fieldObservationMap) {
|
|
678 |
/**
|
|
679 |
* @param state
|
|
680 |
* @param ecoFactId
|
|
681 |
* @param derivedUnitMap
|
|
682 |
* @param type
|
|
683 |
* @return
|
|
684 |
*/
|
|
685 |
private DerivedUnitFacade getDerivedUnit(AlgaTerraImportState state, int ecoFactId, Map<String, DerivedUnit> derivedUnitMap, DerivedUnitType type) {
|
543 |
686 |
String key = String.valueOf(ecoFactId);
|
544 |
|
FieldObservation fieldObservation = fieldObservationMap.get(key);
|
545 |
|
if (fieldObservation == null){
|
546 |
|
fieldObservation = FieldObservation.NewInstance();
|
547 |
|
|
548 |
|
fieldObservationMap.put(key, fieldObservation);
|
|
687 |
DerivedUnit derivedUnit = derivedUnitMap.get(key);
|
|
688 |
DerivedUnitFacade facade;
|
|
689 |
if (derivedUnit == null){
|
|
690 |
facade = DerivedUnitFacade.NewInstance(type);
|
|
691 |
derivedUnitMap.put(key, derivedUnit);
|
|
692 |
}else{
|
|
693 |
try {
|
|
694 |
facade = DerivedUnitFacade.NewInstance(derivedUnit);
|
|
695 |
} catch (DerivedUnitFacadeNotSupportedException e) {
|
|
696 |
logger.error(e.getMessage());
|
|
697 |
facade = DerivedUnitFacade.NewInstance(type);
|
|
698 |
}
|
549 |
699 |
}
|
550 |
700 |
|
551 |
|
return fieldObservation;
|
|
701 |
return facade;
|
552 |
702 |
}
|
553 |
703 |
|
554 |
704 |
private Feature makeFeature(DerivedUnitType type) {
|
... | ... | |
616 |
766 |
result.put(nameSpace, objectMap);
|
617 |
767 |
|
618 |
768 |
//field observation map map
|
619 |
|
nameSpace = AlgaTerraSpecimenImport.FIELD_OBSERVATION_NAMESPACE;
|
|
769 |
nameSpace = AlgaTerraSpecimenImport.ECO_FACT_NAMESPACE;
|
620 |
770 |
cdmClass = FieldObservation.class;
|
621 |
771 |
idSet = taxonIdSet;
|
622 |
772 |
Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
|
latest updates for ALgaterra import