Project

General

Profile

« Previous | Next » 

Revision ef04f11c

Added by Andreas Müller almost 10 years ago

latest updates for ALgaterra import

View differences:

app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraImportState.java
10 10

  
11 11
package eu.etaxonomy.cdm.io.algaterra;
12 12

  
13
import java.util.HashMap;
14
import java.util.Map;
15
import java.util.UUID;
16

  
13 17
import org.apache.log4j.Logger;
14 18

  
15 19
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState;
20
import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
16 21

  
17 22
/**
18 23
 * @author a.mueller
......
25 30
	private static final Logger logger = Logger.getLogger(AlgaTerraImportState.class);
26 31

  
27 32
	private boolean specimenVocabulariesCreated = false;
33
	private boolean currentFieldObservationNotNew = false;
34
	private Map<String, UUID> parameterFeatureUuidMap = new HashMap<String, UUID>(); 
28 35
	
29 36
	public AlgaTerraImportState(AlgaTerraImportConfigurator config) {
30 37
		super(config);
......
42 49
		this.specimenVocabulariesCreated = specimenVocabulariesCreated;
43 50
	}
44 51

  
45
    
52
	public boolean isCurrentFieldObservationNotNew() {
53
		return currentFieldObservationNotNew;
54
	}
55

  
56
	public void setCurrentFieldObservationNotNew(
57
			boolean currentFieldObservationNotNew) {
58
		this.currentFieldObservationNotNew = currentFieldObservationNotNew;
59
	}
60

  
61
	public UUID getParameterFeatureUuid(String key) {
62
		return parameterFeatureUuidMap.get(key);
63
	}
64

  
65
	public void putParameterFeatureUuid(String key, UUID parameterFeatureUuid) {
66
		this.parameterFeatureUuidMap.put(key, parameterFeatureUuid);
67
	}
68

  
46 69
}
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraSpecimenImport.java
26 26

  
27 27
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
28 28
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
29
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeNotSupportedException;
29 30
import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraSpecimenImportValidator;
30 31
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase;
31 32
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
......
41 42
import eu.etaxonomy.cdm.model.common.Marker;
42 43
import eu.etaxonomy.cdm.model.common.MarkerType;
43 44
import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
45
import eu.etaxonomy.cdm.model.common.TermVocabulary;
44 46
import eu.etaxonomy.cdm.model.common.TimePeriod;
45 47
import eu.etaxonomy.cdm.model.description.CategoricalData;
46 48
import eu.etaxonomy.cdm.model.description.DescriptionBase;
47 49
import eu.etaxonomy.cdm.model.description.Feature;
48 50
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
51
import eu.etaxonomy.cdm.model.description.MeasurementUnit;
52
import eu.etaxonomy.cdm.model.description.Modifier;
53
import eu.etaxonomy.cdm.model.description.QuantitativeData;
49 54
import eu.etaxonomy.cdm.model.description.SpecimenDescription;
50 55
import eu.etaxonomy.cdm.model.description.State;
56
import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
57
import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
51 58
import eu.etaxonomy.cdm.model.description.TaxonDescription;
52 59
import eu.etaxonomy.cdm.model.description.TextData;
53 60
import eu.etaxonomy.cdm.model.location.NamedArea;
......
57 64
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
58 65
import eu.etaxonomy.cdm.model.name.BotanicalName;
59 66
import eu.etaxonomy.cdm.model.name.Rank;
67
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
60 68
import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
61 69
import eu.etaxonomy.cdm.model.reference.Reference;
62 70
import eu.etaxonomy.cdm.model.taxon.Taxon;
......
72 80
public class AlgaTerraSpecimenImport  extends BerlinModelImportBase {
73 81
	private static final Logger logger = Logger.getLogger(AlgaTerraSpecimenImport.class);
74 82

  
75
	public static final String FIELD_OBSERVATION_NAMESPACE = "FieldObservation";
83
	public static final String ECO_FACT_NAMESPACE = "EcoFact";
76 84
	public static final String TERMS_NAMESPACE = "ALGA_TERRA_TERMS";
77 85
	
78
	//move to transformrer
86
	//TODO move to transformrer
79 87
	final static UUID uuidMarkerAlkalinity = UUID.fromString("e52d0ea2-0c1f-4d95-ae6d-e21ab317c594");  
80 88
	final static UUID uuidRefSystemGps = UUID.fromString("c23e4928-c137-4e4a-b6ab-b430da3d0b94");  
81 89
	final static UUID uuidFeatureSpecimenCommunity = UUID.fromString("3ff5b1ab-3999-4b5a-b8f7-01fd2f6c12c7");
82 90
	final static UUID uuidFeatureAdditionalData = UUID.fromString("0ac82ab8-2c2b-4953-98eb-a9f718eb9c57");
91
	final static UUID uuidFeatureHabitatExplanation = UUID.fromString("6fe32295-61a3-44fc-9fcf-a85790ea888f");
83 92
	
84 93
	final static UUID uuidVocAlgaTerraClimate = UUID.fromString("b0a677c6-8bb6-43f4-b1b8-fc377a10feb5");
85 94
	final static UUID uuidVocAlgaTerraHabitat = UUID.fromString("06f30114-e19c-4e7d-a8e5-5488c41fcbc5");
......
89 98
	final static UUID uuidFeatureAlgaTerraHabitat = UUID.fromString("7def3fc2-cdc5-4739-8e13-62edbd053415");
90 99
	final static UUID uuidFeatureAlgaTerraLifeForm = UUID.fromString("9b657901-1b0d-4a2a-8d21-dd8c1413e2e6");
91 100
	
101
	final static UUID uuidVocParameter = UUID.fromString("45888b40-5bbb-4293-aa1e-02479796cd7c");
102
	final static UUID uuidStatMeasureSingleValue = UUID.fromString("eb4c3d98-4d4b-4c37-8eb4-17315ce79920");
103
	final static UUID uuidMeasurementValueModifier = UUID.fromString("0218a7a3-f6c0-4d06-a4f8-6b50b73aef5e");
92 104
	
105
	final static UUID uuidModifierLowerThan = UUID.fromString("2b500085-6bef-4003-b6ea-e0ad0237d79d");
106
	final static UUID uuidModifierGreaterThan = UUID.fromString("828df49d-c745-48f7-b083-0ada43356c34");
93 107
	
94 108
	private static int modCount = 5000;
95 109
	private static final String pluralString = "specimen and observation";
......
154 168
		Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
155 169
		
156 170
		Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
157
		Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>) partitioner.getObjectMap(FIELD_OBSERVATION_NAMESPACE);
171
		Map<String, DerivedUnit> ecoFactMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(ECO_FACT_NAMESPACE);
158 172
		
159 173
		ResultSet rs = partitioner.getResultSet();
160 174

  
......
178 192
					Reference<?> sourceRef = state.getTransactionalSourceReference();
179 193
				
180 194
					//facade
181
					FieldObservation fieldObservation = getFieldObservation(ecoFactId, fieldObservationMap);
182 195
					DerivedUnitType type = makeDerivedUnitType(recordBasis);
183
					DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(type, fieldObservation);
184

  
196
					DerivedUnitFacade facade = getDerivedUnit(state, ecoFactId, ecoFactMap, type);
197
					
185 198
					//field observation
186 199
					handleSingleSpecimen(rs, facade, state);
187 200
					
201
					state.setCurrentFieldObservationNotNew(false);
202
					
188 203
					//description element
189 204
					TaxonDescription taxonDescription = getTaxonDescription(state, newTaxonId, taxonMap, factId, sourceRef);
190 205
					IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
......
295 310

  
296 311

  
297 312
	private void handleSingleSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state) throws SQLException {
298
       	//CollectionFk, Collector, AltitudeMethod, 
299
    	//ISOCountrySub, CreatedWhen/Who/Updated/who
300
    	
301
    	//P1-10Value/Unit/Parameter/Method
302
		
313
		//FIXME missing fields #3084, #3085, #3080
303 314
		try {
304 315
			Object alkalinityFlag = rs.getBoolean("AlkalinityFlag");
305 316
			
......
329 340
			
330 341
			
331 342
			
332
			FieldObservation fieldObservation = facade.innerFieldObservation();
343
			FieldObservation fieldObservation = facade.getFieldObservation(true);
333 344
			
334 345
			//alkalinity marker
335 346
			if (alkalinityFlag != null){
......
376 387
			
377 388
			//habitat, ecology, community, etc.
378 389
			DescriptionBase<?> fieldDescription = getFieldObservationDescription(facade);
390
			
379 391
			addCategoricalValue(state, fieldDescription, climateUuid, uuidFeatureAlgaTerraClimate);
380 392
			addCategoricalValue(state, fieldDescription, habitatUuid, Feature.HABITAT().getUuid());
381 393
			addCategoricalValue(state, fieldDescription, lifeFormUuid, uuidFeatureAlgaTerraLifeForm);
382 394
			
383 395
			if (isNotBlank(habitat)){
384
				//FIXME
385
				facade.setEcology(habitat);  //or use an own feature ??
396
				Feature habitatExplanation = getFeature(state, uuidFeatureHabitatExplanation, "Habitat Explanation", "HabitatExplanation", null, null);
397
				TextData textData = TextData.NewInstance(habitatExplanation);
398
				textData.putText(Language.DEFAULT(), habitat);
399
				getFieldObservationDescription(facade).addElement(textData);
386 400
			}
387 401
			if (isNotBlank(community)){
388 402
				Feature communityFeature = getFeature(state, uuidFeatureSpecimenCommunity, "Community", "The community of a specimen (e.g. other algae in the same sample)", null, null);
......
407 421
			makeAreas(state, rs, facade);
408 422
			
409 423
			//parameters
410
			//TODO
424
			makeParameter(state, rs, getFieldObservationDescription(facade));
411 425
			
412 426
			//collection
413 427
			String voucher = rs.getString("Voucher");
......
422 436
			//TODO id, created for fact +  ecoFact
423 437
			//    	this.doIdCreatedUpdatedNotes(state, descriptionElement, rs, id, namespace);
424 438
		
425
		} catch (IllegalArgumentException e) {
426
			throw e;
439
		} catch (Exception e) {
440
			throw new RuntimeException(e);
427 441
		}
428 442
    	
429 443
	}
430 444

  
431 445

  
446
	private void makeParameter(AlgaTerraImportState state, ResultSet rs, DescriptionBase<?> descriptionBase) throws SQLException {
447
		for (int i = 1; i <= 10; i++){
448
			String valueStr = rs.getString(String.format("P%dValue", i));
449
			String unitStr = rs.getString(String.format("P%dUnit", i));
450
			String parameter = rs.getString(String.format("P%dParameter", i));
451
			String method = rs.getString(String.format("P%dMethod", i));
452
			
453
			//method
454
			if (StringUtils.isNotBlank(method)){
455
				logger.warn("Methods not yet handled: " + method);
456
			}
457
			//parameter
458
			TermVocabulary<Feature> vocParameter = getVocabulary(uuidVocParameter, "Feature vocabulary for AlgaTerra measurement parameters", "Parameters", null, null, false, Feature.COMMON_NAME());
459
			if (StringUtils.isNotBlank(parameter)){
460
				UUID featureUuid = getParameterFeatureUuid(state, parameter);
461
				Feature feature = getFeature(state, featureUuid, parameter, parameter, null, vocParameter);
462
				QuantitativeData quantData = QuantitativeData.NewInstance(feature);
463
				
464
				//unit
465
				MeasurementUnit unit = getMeasurementUnit(state, unitStr);
466
				quantData.setUnit(unit);
467
				try {
468
					
469
					Set<Modifier> valueModifier = new HashSet<Modifier>();
470
					valueStr = normalizeAndModifyValue(state, valueStr, valueModifier);
471
					//value
472
					Float valueFlt = Float.valueOf(valueStr);  //TODO maybe change model to Double ??
473
					
474
					StatisticalMeasure measureSingleValue = getStatisticalMeasure(state, uuidStatMeasureSingleValue, "Value", "Single measurement value", null, null);
475
					StatisticalMeasurementValue value = StatisticalMeasurementValue.NewInstance(measureSingleValue, valueFlt); 
476
					quantData.addStatisticalValue(value);
477
					descriptionBase.addElement(quantData);
478
					
479
				} catch (NumberFormatException e) {
480
					logger.warn(String.format("Value '%s' can't be converted to double. Parameter %s not imported.", valueStr, parameter));
481
				}
482
			}else if (isNotBlank(valueStr) || isNotBlank(unitStr) ){
483
				logger.warn("There is value or unit without parameter: " + i);
484
			}
485
			
486
			
487
		}
488
		
489
	}
490

  
491
	private String normalizeAndModifyValue(AlgaTerraImportState state, String valueStr, Set<Modifier> valueModifier) {
492
		valueStr = valueStr.replace(",", ".");
493
		if (valueStr.startsWith("<")){
494
			TermVocabulary<Modifier> measurementValueModifierVocabulary = getVocabulary(uuidMeasurementValueModifier, "Measurement value modifier", "Measurement value modifier", null, null, false, Modifier.NewInstance());
495
			Modifier modifier = getModifier(state, uuidModifierLowerThan, "Lower", "Lower than the given measurement value", "<", measurementValueModifierVocabulary);
496
			valueModifier.add(modifier);
497
			valueStr = valueStr.replace("<", "");
498
		}
499
		if (valueStr.startsWith(">")){
500
			TermVocabulary<Modifier> measurementValueModifierVocabulary = getVocabulary(uuidMeasurementValueModifier, "Measurement value modifier", "Measurement value modifier", null, null, false, Modifier.NewInstance());
501
			Modifier modifier = getModifier(state, uuidModifierGreaterThan, "Lower", "Lower than the given measurement value", "<", measurementValueModifierVocabulary);
502
			valueModifier.add(modifier);
503
			valueStr = valueStr.replace(">", "");
504
		}
505
		return valueStr;
506
	}
507

  
508

  
509

  
510
	private UUID getParameterFeatureUuid(AlgaTerraImportState state, String key) {
511
		//TODO define some UUIDs in Transformer
512
		UUID uuid = state.getParameterFeatureUuid(key);
513
		if (uuid == null){
514
			uuid = UUID.randomUUID();
515
			state.putParameterFeatureUuid(key, uuid);
516
		}
517
		return uuid;
518
	}
519

  
520

  
521

  
522
	/**
523
	 * TODO move to InputTransformerBase
524
	 * @param state
525
	 * @param unitStr
526
	 * @return
527
	 */
528
	private MeasurementUnit getMeasurementUnit(AlgaTerraImportState state, String unitStr) {
529
		MeasurementUnit result = null;
530
		if (StringUtils.isNotBlank(unitStr)){
531
			UUID uuidMeasurementUnitMgL = UUID.fromString("7ac302c5-3cbd-4334-964a-bf5d11eb9ead");
532
			UUID uuidMeasurementUnitMolMol = UUID.fromString("96b78d78-3e49-448f-8100-e7779b71dd53");
533
			UUID uuidMeasurementUnitMicroMolSiL = UUID.fromString("2cb8bc85-a4af-42f1-b80b-34c36c9f75d4");
534
			UUID uuidMeasurementUnitMicroMolL = UUID.fromString("a631f62e-377e-405c-bd1a-76885b13a72b");
535
			UUID uuidMeasurementUnitDegreeC = UUID.fromString("55222aec-d5be-413e-8db7-d9a48c316c6c");
536
			UUID uuidMeasurementUnitPercent = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
537
			UUID uuidMeasurementUnitCm = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
538
			UUID uuidMeasurementUnitMicroSiCm = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
539
			
540
			
541
			if (unitStr.equalsIgnoreCase("mg/L")){
542
				return  getMeasurementUnit(state, uuidMeasurementUnitMgL, unitStr, unitStr, unitStr, null);
543
			}else if (unitStr.equalsIgnoreCase("mol/mol")){
544
				return result = getMeasurementUnit(state, uuidMeasurementUnitMolMol, unitStr, unitStr, unitStr, null);
545
			}else if (unitStr.equalsIgnoreCase("\u00B5mol Si/L")){   //µmol Si/L
546
				return getMeasurementUnit(state, uuidMeasurementUnitMicroMolSiL, unitStr, unitStr, unitStr, null);
547
			}else if (unitStr.equalsIgnoreCase("\u00B5mol/L")){		//µmol/L
548
				return getMeasurementUnit(state, uuidMeasurementUnitMicroMolL, unitStr, unitStr, unitStr, null);
549
			}else if (unitStr.equalsIgnoreCase("\u00B0C")){               //°C
550
				return getMeasurementUnit(state, uuidMeasurementUnitDegreeC, unitStr, unitStr, unitStr, null);
551
			}else if (unitStr.equalsIgnoreCase("%")){
552
				return getMeasurementUnit(state, uuidMeasurementUnitPercent, unitStr, unitStr, unitStr, null);
553
			}else if (unitStr.equalsIgnoreCase("cm")){
554
				return getMeasurementUnit(state, uuidMeasurementUnitCm, unitStr, unitStr, unitStr, null);
555
			}else if (unitStr.equalsIgnoreCase("\u00B5S/cm")){   //µS/cm
556
				return getMeasurementUnit(state, uuidMeasurementUnitMicroSiCm, unitStr, unitStr, unitStr, null);
557
			}else{
558
				logger.warn("MeasurementUnit was not recognized");
559
				return null;
560
			}
561
		}else{
562
			return null;
563
		}
564
	}
565

  
566

  
567

  
432 568
	private void addCategoricalValue(AlgaTerraImportState importState, DescriptionBase description, String uuidTerm, UUID featureUuid) {
433 569
		if (uuidTerm != null){
434 570
			State state = this.getStateTerm(importState, UUID.fromString(uuidTerm));
......
539 675
		}
540 676
	}
541 677

  
542
	private FieldObservation getFieldObservation(int ecoFactId, Map<String, FieldObservation> fieldObservationMap) {
678
	/**
679
	 * @param state
680
	 * @param ecoFactId
681
	 * @param derivedUnitMap
682
	 * @param type 
683
	 * @return
684
	 */
685
	private DerivedUnitFacade getDerivedUnit(AlgaTerraImportState state, int ecoFactId, Map<String, DerivedUnit> derivedUnitMap, DerivedUnitType type) {
543 686
		String key = String.valueOf(ecoFactId);
544
		FieldObservation fieldObservation = fieldObservationMap.get(key);
545
		if (fieldObservation == null){
546
			fieldObservation = FieldObservation.NewInstance();
547
			
548
			fieldObservationMap.put(key, fieldObservation);
687
		DerivedUnit derivedUnit = derivedUnitMap.get(key);
688
		DerivedUnitFacade facade;
689
		if (derivedUnit == null){
690
			facade = DerivedUnitFacade.NewInstance(type);
691
			derivedUnitMap.put(key, derivedUnit);
692
		}else{
693
			try {
694
				facade = DerivedUnitFacade.NewInstance(derivedUnit);
695
			} catch (DerivedUnitFacadeNotSupportedException e) {
696
				logger.error(e.getMessage());
697
				facade = DerivedUnitFacade.NewInstance(type);
698
			}
549 699
		}
550 700
		
551
		return fieldObservation;
701
		return facade;
552 702
	}
553 703
	
554 704
	private Feature makeFeature(DerivedUnitType type) {
......
616 766
			result.put(nameSpace, objectMap);
617 767

  
618 768
			//field observation map map
619
			nameSpace = AlgaTerraSpecimenImport.FIELD_OBSERVATION_NAMESPACE;
769
			nameSpace = AlgaTerraSpecimenImport.ECO_FACT_NAMESPACE;
620 770
			cdmClass = FieldObservation.class;
621 771
			idSet = taxonIdSet;
622 772
			Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);

Also available in: Unified diff