app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraSpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.algaterra;
  11
  12 import java.net.URI;
  13 import java.sql.Date;
  14 import java.sql.ResultSet;
  15 import java.sql.SQLException;
  16 import java.util.Map;
  17 import java.util.Set;
  18 import java.util.UUID;
  19
  20 import org.apache.commons.lang.StringUtils;
  21 import org.apache.log4j.Logger;
  22 import org.springframework.transaction.TransactionStatus;
  23
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase;
  26 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonNameImport;
  27 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  28 import eu.etaxonomy.cdm.io.common.Source;
  29 import eu.etaxonomy.cdm.model.agent.Team;
  30 import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
  31 import eu.etaxonomy.cdm.model.common.TimePeriod;
  32 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  33 import eu.etaxonomy.cdm.model.description.Feature;
  34 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  35 import eu.etaxonomy.cdm.model.description.State;
  36 import eu.etaxonomy.cdm.model.location.NamedArea;
  37 import eu.etaxonomy.cdm.model.location.Point;
  38 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  39 import eu.etaxonomy.cdm.model.location.TdwgArea;
  40 import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
  41 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  42 import eu.etaxonomy.cdm.model.occurrence.Collection;
  43
  44 /**
  45  * @author a.mueller
  46  * @created 12.09.2012
  47  */
  48 public abstract class AlgaTerraSpecimenImportBase extends BerlinModelImportBase{
  49         private static final Logger logger = Logger.getLogger(AlgaTerraSpecimenImportBase.class);
  50
  51         public static final String ECO_FACT_NAMESPACE = "EcoFact";
  52         public static final String TYPE_SPECIMEN_NAMESPACE = "TypeSpecimen";
  53         public static final String TERMS_NAMESPACE = "ALGA_TERRA_TERMS";
  54
  55         //TODO move to transformrer
  56         final static UUID uuidMarkerAlkalinity = UUID.fromString("e52d0ea2-0c1f-4d95-ae6d-e21ab317c594");
  57         final static UUID uuidRefSystemGps = UUID.fromString("c23e4928-c137-4e4a-b6ab-b430da3d0b94");
  58         final static UUID uuidFeatureSpecimenCommunity = UUID.fromString("3ff5b1ab-3999-4b5a-b8f7-01fd2f6c12c7");
  59         final static UUID uuidFeatureAdditionalData = UUID.fromString("0ac82ab8-2c2b-4953-98eb-a9f718eb9c57");
  60         final static UUID uuidFeatureHabitatExplanation = UUID.fromString("6fe32295-61a3-44fc-9fcf-a85790ea888f");
  61
  62         final static UUID uuidVocAlgaTerraClimate = UUID.fromString("b0a677c6-8bb6-43f4-b1b8-fc377a10feb5");
  63         final static UUID uuidVocAlgaTerraHabitat = UUID.fromString("06f30114-e19c-4e7d-a8e5-5488c41fcbc5");
  64         final static UUID uuidVocAlgaTerraLifeForm = UUID.fromString("3c0b194e-809c-4b42-9498-6ff034066ed7");
  65
  66         final static UUID uuidFeatureAlgaTerraClimate = UUID.fromString("8754674c-9ab9-4f28-95f1-91eeee2314ee");
  67         final static UUID uuidFeatureAlgaTerraHabitat = UUID.fromString("7def3fc2-cdc5-4739-8e13-62edbd053415");
  68         final static UUID uuidFeatureAlgaTerraLifeForm = UUID.fromString("9b657901-1b0d-4a2a-8d21-dd8c1413e2e6");
  69
  70         final static UUID uuidVocParameter = UUID.fromString("45888b40-5bbb-4293-aa1e-02479796cd7c");
  71         final static UUID uuidStatMeasureSingleValue = UUID.fromString("eb4c3d98-4d4b-4c37-8eb4-17315ce79920");
  72         final static UUID uuidMeasurementValueModifier = UUID.fromString("0218a7a3-f6c0-4d06-a4f8-6b50b73aef5e");
  73
  74         final static UUID uuidModifierLowerThan = UUID.fromString("2b500085-6bef-4003-b6ea-e0ad0237d79d");
  75         final static UUID uuidModifierGreaterThan = UUID.fromString("828df49d-c745-48f7-b083-0ada43356c34");
  76
  77
  78         /**
  79          * Creates the vocabularies and the features for Climate, Habitat and Lifeform
  80          * @param state
  81          * @throws SQLException
  82          */
  83         protected void makeVocabulariesAndFeatures(AlgaTerraImportState state) throws SQLException {
  84                 String abbrevLabel = null;
  85                 URI uri = null;
  86
  87                 if (! state.isSpecimenVocabulariesCreated()){
  88
  89                         TransactionStatus txStatus = this.startTransaction();
  90
  91                         boolean isOrdered = true;
  92                         OrderedTermVocabulary<State> climateVoc = (OrderedTermVocabulary)getVocabulary(uuidVocAlgaTerraClimate, "Climate", "Climate", abbrevLabel, uri, isOrdered, null);
  93                         OrderedTermVocabulary<State> habitatVoc = (OrderedTermVocabulary)getVocabulary(uuidVocAlgaTerraHabitat, "Habitat", "Habitat", abbrevLabel, uri, isOrdered, null);
  94                         OrderedTermVocabulary<State> lifeformVoc = (OrderedTermVocabulary)getVocabulary(uuidVocAlgaTerraLifeForm, "Lifeform", "Lifeform", abbrevLabel, uri, isOrdered, null);
  95
  96
  97                         Feature feature = getFeature(state, uuidFeatureAlgaTerraClimate, "Climate","Climate", null, null);
  98                         feature.setSupportsCategoricalData(true);
  99
 100                         feature = getFeature(state, uuidFeatureAlgaTerraLifeForm, "LifeForm","LifeForm", null, null);
 101                         feature.setSupportsCategoricalData(true);
 102
 103                         feature = Feature.HABITAT();
 104                         feature.setSupportsCategoricalData(true);
 105                         getTermService().saveOrUpdate(feature);
 106
 107                         Source source = state.getAlgaTerraConfigurator().getSource();
 108
 109                         String climateSql = "SELECT * FROM EcoClimate";
 110                         ResultSet rs = source.getResultSet(climateSql);
 111                         while (rs.next()){
 112                                 String climate = rs.getString("Climate");
 113                                 String description = rs.getString("Description");
 114                                 Integer id = rs.getInt("ClimateId");
 115                                 UUID uuid = UUID.fromString(rs.getString("UUID"));
 116                                 State stateTerm = getStateTerm(state, uuid, climate, description, null, climateVoc);
 117                                 addOriginalSource(stateTerm, id.toString(), "EcoClimate", state.getTransactionalSourceReference());
 118                                 getTermService().saveOrUpdate(stateTerm);
 119                         }
 120
 121                         String habitatSql = "SELECT * FROM EcoHabitat";
 122                         rs = source.getResultSet(habitatSql);
 123                         while (rs.next()){
 124                                 String habitat = rs.getString("Habitat");
 125                                 String description = rs.getString("Description");
 126                                 Integer id = rs.getInt("HabitatId");
 127                                 UUID uuid = UUID.fromString(rs.getString("UUID"));
 128                                 State stateTerm = getStateTerm(state, uuid, habitat, description, null, habitatVoc);
 129                                 addOriginalSource(stateTerm, id.toString(), "EcoHabitat", state.getTransactionalSourceReference());
 130                                 getTermService().saveOrUpdate(stateTerm);
 131                         }
 132
 133                         String lifeformSql = "SELECT * FROM EcoLifeForm";
 134                         rs = source.getResultSet(lifeformSql);
 135                         while (rs.next()){
 136                                 String lifeform = rs.getString("LifeForm");
 137                                 String description = rs.getString("Description");
 138                                 Integer id = rs.getInt("LifeFormId");
 139                                 UUID uuid = UUID.fromString(rs.getString("UUID"));
 140                                 State stateTerm = getStateTerm(state, uuid, lifeform, description, null, lifeformVoc);
 141                                 addOriginalSource(stateTerm, id.toString(), "EcoLifeForm", state.getTransactionalSourceReference());
 142                                 getTermService().saveOrUpdate(stateTerm);
 143                         }
 144
 145                         this.commitTransaction(txStatus);
 146
 147                         state.setSpecimenVocabulariesCreated(true);
 148                 }
 149
 150         }
 151
 152         protected String getLocalityString(){
 153                 return "Locality";
 154         }
 155
 156         protected void handleSingleSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state, ResultSetPartitioner partitioner) throws SQLException {
 157                 //FIXME missing fields #3084, #3085, #3080
 158                 try {
 159
 160                         String locality = rs.getString(getLocalityString());
 161                         Double latitude = nullSafeDouble(rs, "Latitude");
 162                         Double longitude = nullSafeDouble(rs, "Longitude");
 163                         Integer errorRadius = nullSafeInt(rs,"Prec");
 164                         String geoCodeMethod = rs.getString("GeoCodeMethod");
 165
 166                         Integer altitude = nullSafeInt(rs, "Altitude");
 167                         Integer lowerAltitude = nullSafeInt(rs,"AltitudeLowerValue");
 168                         String altitudeUnit = rs.getString("AltitudeUnit");
 169                         Double depth = nullSafeDouble(rs, "Depth");
 170                         Double depthLow = nullSafeDouble(rs, "DepthLow");
 171
 172                         String collectorsNumber = rs.getString("CollectorsNumber");
 173                         Date collectionDateStart = rs.getDate("CollectionDate");
 174                         Date collectionDateEnd = rs.getDate("CollectionDateEnd");
 175
 176                         Integer collectionFk = nullSafeInt(rs,"CollectionFk");
 177
 178
 179                         //location
 180                         facade.setLocality(locality);
 181
 182                         //exact location
 183                         ReferenceSystem referenceSystem = makeRefrenceSystem(geoCodeMethod, state);
 184                         if (longitude != null || latitude != null || referenceSystem != null || errorRadius != null){
 185                                 Point exactLocation = Point.NewInstance(longitude, latitude, referenceSystem, errorRadius);
 186                                 facade.setExactLocation(exactLocation);
 187                         }
 188
 189                         //altitude, depth
 190                         if (StringUtils.isNotBlank(altitudeUnit) && ! altitudeUnit.trim().equalsIgnoreCase("m")){
 191                                 logger.warn("Altitude unit is not [m] but: " +  altitudeUnit);
 192                         }
 193                         if ( altitude != null){
 194                                 if (lowerAltitude == null){
 195                                         facade.setAbsoluteElevation(altitude);
 196                                 }else{
 197                                         if (! facade.isEvenDistance(lowerAltitude, altitude)){
 198                                                 //FIXME there is a ticket for this
 199                                                 altitude = altitude + 1;
 200                                                 logger.info("Current implementation of altitude does not allow uneven distances");
 201                                         }
 202                                         facade.setAbsoluteElevationRange(lowerAltitude,altitude);
 203                                 }
 204                         }
 205                         if ( depth != null){
 206                                 //FIXME needs model change to accept double #3072
 207                                 Integer intDepth = depth.intValue();
 208                                 if (depthLow == null){
 209                                         facade.setDistanceToWaterSurface(intDepth);
 210                                 }else{
 211                                         //FIXME range not yet in model #3074
 212                                         facade.setDistanceToWaterSurface(intDepth);
 213                                 }
 214                         }
 215
 216                         //field
 217                         facade.setFieldNumber(collectorsNumber);
 218                         TimePeriod gatheringPeriod = TimePeriod.NewInstance(collectionDateStart, collectionDateEnd);
 219                         facade.setGatheringPeriod(gatheringPeriod);
 220                         handleCollectorTeam(state, facade, rs);
 221
 222                         //areas
 223                         makeAreas(state, rs, facade);
 224
 225                         //collection
 226                         if (collectionFk != null){
 227                                 Collection subCollection = state.getRelatedObject(AlgaTerraCollectionImport.NAMESPACE_SUBCOLLECTION, String.valueOf(collectionFk), Collection.class);
 228                                 if (subCollection != null){
 229                                         facade.setCollection(subCollection);
 230                                 }else{
 231                                         Collection collection = state.getRelatedObject(AlgaTerraCollectionImport.NAMESPACE_COLLECTION, String.valueOf(collectionFk), Collection.class);
 232                                         facade.setCollection(collection);
 233                                 }
 234                         }
 235
 236                         //notes
 237                         //TODO is this an annotation on field observation or on the derived unit?
 238
 239                         //TODO id, created for fact +  ecoFact
 240                         //      this.doIdCreatedUpdatedNotes(state, descriptionElement, rs, id, namespace);
 241
 242                 } catch (Exception e) {
 243                         throw new RuntimeException(e);
 244                 }
 245
 246         }
 247
 248         protected DescriptionBase getFieldObservationDescription(DerivedUnitFacade facade) {
 249                 Set<DescriptionBase> descriptions = facade.innerFieldObservation().getDescriptions();
 250                 for (DescriptionBase desc : descriptions){
 251                         if (desc.isImageGallery() == false){
 252                                 return desc;
 253                         }
 254                 }
 255                 SpecimenDescription specDesc = SpecimenDescription.NewInstance(facade.innerFieldObservation());
 256                 descriptions.add(specDesc);
 257                 return specDesc;
 258         }
 259
 260
 261         private void makeAreas(AlgaTerraImportState state, ResultSet rs, DerivedUnitFacade facade) throws SQLException {
 262                 Object gazetteerId = rs.getObject("GazetteerId");
 263                 if (gazetteerId != null){
 264                         //TDWG
 265                         NamedArea tdwgArea;
 266                         String tdwg4 = rs.getString("L4Code");
 267                         if (isNotBlank(tdwg4)){
 268                                 tdwgArea = TdwgArea.getAreaByTdwgAbbreviation(tdwg4);
 269                         }else{
 270                                 String tdwg3 = rs.getString("L3Code");
 271                                 if (isNotBlank(tdwg3)){
 272                                         tdwgArea = TdwgArea.getAreaByTdwgAbbreviation(tdwg3);
 273                                 }else{
 274                                         Integer tdwg2 = rs.getInt("L2Code");
 275                                         tdwgArea = TdwgArea.getAreaByTdwgAbbreviation(String.valueOf(tdwg2));
 276                                 }
 277                         }
 278                         if (tdwgArea == null){
 279                                 logger.warn("TDWG area could not be defined for gazetterId: " + gazetteerId);
 280                         }else{
 281                                 facade.addCollectingArea(tdwgArea);
 282                         }
 283
 284                         //Countries
 285                         WaterbodyOrCountry country = null;
 286                         String isoCountry = rs.getString("ISOCountry");
 287                         String countryStr = rs.getString("Country");
 288                         if (isNotBlank(isoCountry)){
 289                                 country = WaterbodyOrCountry.getWaterbodyOrCountryByIso3166A2(isoCountry);
 290                         }else if (isNotBlank(countryStr)){
 291                                 logger.warn("Country exists but no ISO code");
 292                         }
 293                         if (country == null){
 294                                 logger.warn("Country does not exist for GazetteerID " + gazetteerId);
 295                         }else{
 296                                 facade.setCountry(country);
 297                         }
 298
 299                 }
 300
 301                 //Waterbody
 302                 WaterbodyOrCountry waterbody = null;
 303                 String waterbodyStr = rs.getString("WaterBody");
 304                 if (isNotBlank(waterbodyStr)){
 305                         if (waterbodyStr.equals("Atlantic Ocean")){
 306                                 waterbody = WaterbodyOrCountry.ATLANTICOCEAN();
 307                         }else{
 308                                 logger.warn("Waterbody not recognized: " + waterbody);
 309                         }
 310                         if (waterbody != null){
 311                                 facade.addCollectingArea(waterbody);
 312                         }
 313                 }
 314
 315
 316                 //countries sub
 317                 //TODO -> SpecimenImport (not existing in TypeSpecimen)
 318         }
 319
 320
 321
 322
 323         private ReferenceSystem makeRefrenceSystem(String geoCodeMethod, AlgaTerraImportState state) {
 324                 if (StringUtils.isBlank(geoCodeMethod)){
 325                         return null;
 326                 }else if(geoCodeMethod.startsWith("GPS")){
 327                         getReferenceSystem(state, uuidRefSystemGps, "GPS", "GPS", "GPS", ReferenceSystem.GOOGLE_EARTH().getVocabulary());
 328                         return ReferenceSystem.WGS84();
 329                 }else if(geoCodeMethod.startsWith("Google")){
 330                         return ReferenceSystem.GOOGLE_EARTH();
 331                 }else if(geoCodeMethod.startsWith("Map")){
 332                         logger.warn("Reference system " +  geoCodeMethod +  " not yet supported.");
 333                         return null;
 334                 }else if(geoCodeMethod.startsWith("WikiProjekt Georeferenzierung") || geoCodeMethod.startsWith("http://toolserver.org/~geohack/geohack.php") ){
 335                         return ReferenceSystem.WGS84();
 336                 }else {
 337                         logger.warn("Reference system " +  geoCodeMethod +  " not yet supported.");
 338                         return null;
 339                 }
 340         }
 341
 342
 343
 344
 345         private void handleCollectorTeam(AlgaTerraImportState state, DerivedUnitFacade facade, ResultSet rs) throws SQLException {
 346                 // FIXME parsen
 347                 String collector = rs.getString("Collector");
 348                 Team team = Team.NewTitledInstance(collector, collector);
 349                 facade.setCollector(team);
 350         }
 351
 352
 353
 354
 355 }