app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraSpecimenImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.algaterra;
  11
  12 import java.net.URI;
  13 import java.sql.Date;
  14 import java.sql.ResultSet;
  15 import java.sql.SQLException;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.Map;
  19 import java.util.Set;
  20 import java.util.UUID;
  21
  22 import org.apache.commons.lang.StringUtils;
  23 import org.apache.log4j.Logger;
  24 import org.springframework.stereotype.Component;
  25 import org.springframework.transaction.TransactionStatus;
  26
  27 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  28 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  29 import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraSpecimenImportValidator;
  30 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase;
  31 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
  32 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState;
  33 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonImport;
  34 import eu.etaxonomy.cdm.io.common.IOValidator;
  35 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  36 import eu.etaxonomy.cdm.io.common.Source;
  37 import eu.etaxonomy.cdm.model.agent.Team;
  38 import eu.etaxonomy.cdm.model.common.CdmBase;
  39 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  40 import eu.etaxonomy.cdm.model.common.Language;
  41 import eu.etaxonomy.cdm.model.common.Marker;
  42 import eu.etaxonomy.cdm.model.common.MarkerType;
  43 import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
  44 import eu.etaxonomy.cdm.model.common.TimePeriod;
  45 import eu.etaxonomy.cdm.model.description.CategoricalData;
  46 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  47 import eu.etaxonomy.cdm.model.description.Feature;
  48 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  49 import eu.etaxonomy.cdm.model.description.SpecimenDescription;
  50 import eu.etaxonomy.cdm.model.description.State;
  51 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  52 import eu.etaxonomy.cdm.model.description.TextData;
  53 import eu.etaxonomy.cdm.model.location.NamedArea;
  54 import eu.etaxonomy.cdm.model.location.Point;
  55 import eu.etaxonomy.cdm.model.location.ReferenceSystem;
  56 import eu.etaxonomy.cdm.model.location.TdwgArea;
  57 import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
  58 import eu.etaxonomy.cdm.model.name.BotanicalName;
  59 import eu.etaxonomy.cdm.model.name.Rank;
  60 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
  61 import eu.etaxonomy.cdm.model.reference.Reference;
  62 import eu.etaxonomy.cdm.model.taxon.Taxon;
  63 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  64
  65
  66 /**
  67  * @author a.mueller
  68  * @created 20.03.2008
  69  * @version 1.0
  70  */
  71 @Component
  72 public class AlgaTerraSpecimenImport  extends BerlinModelImportBase {
  73         private static final Logger logger = Logger.getLogger(AlgaTerraSpecimenImport.class);
  74
  75         public static final String FIELD_OBSERVATION_NAMESPACE = "FieldObservation";
  76         public static final String TERMS_NAMESPACE = "ALGA_TERRA_TERMS";
  77
  78         //move to transformrer
  79         final static UUID uuidMarkerAlkalinity = UUID.fromString("e52d0ea2-0c1f-4d95-ae6d-e21ab317c594");
  80         final static UUID uuidRefSystemGps = UUID.fromString("c23e4928-c137-4e4a-b6ab-b430da3d0b94");
  81         final static UUID uuidFeatureSpecimenCommunity = UUID.fromString("3ff5b1ab-3999-4b5a-b8f7-01fd2f6c12c7");
  82         final static UUID uuidFeatureAdditionalData = UUID.fromString("0ac82ab8-2c2b-4953-98eb-a9f718eb9c57");
  83
  84         final static UUID uuidVocAlgaTerraClimate = UUID.fromString("b0a677c6-8bb6-43f4-b1b8-fc377a10feb5");
  85         final static UUID uuidVocAlgaTerraHabitat = UUID.fromString("06f30114-e19c-4e7d-a8e5-5488c41fcbc5");
  86         final static UUID uuidVocAlgaTerraLifeForm = UUID.fromString("3c0b194e-809c-4b42-9498-6ff034066ed7");
  87
  88         final static UUID uuidFeatureAlgaTerraClimate = UUID.fromString("8754674c-9ab9-4f28-95f1-91eeee2314ee");
  89         final static UUID uuidFeatureAlgaTerraHabitat = UUID.fromString("7def3fc2-cdc5-4739-8e13-62edbd053415");
  90         final static UUID uuidFeatureAlgaTerraLifeForm = UUID.fromString("9b657901-1b0d-4a2a-8d21-dd8c1413e2e6");
  91
  92
  93
  94         private static int modCount = 5000;
  95         private static final String pluralString = "specimen and observation";
  96         private static final String dbTableName = "Fact";  //??
  97
  98
  99         public AlgaTerraSpecimenImport(){
 100                 super();
 101         }
 102
 103
 104
 105         /* (non-Javadoc)
 106          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery()
 107          */
 108         @Override
 109         protected String getIdQuery(BerlinModelImportState state) {
 110                 String result = " SELECT factId " +
 111                                 " FROM Fact " +
 112                                         " INNER JOIN EcoFact ON Fact.ExtensionFk = EcoFact.EcoFactId " +
 113                                         "INNER JOIN PTaxon ON Fact.PTNameFk = PTaxon.PTNameFk AND Fact.PTRefFk = PTaxon.PTRefFk "
 114                                 + " WHERE FactCategoryFk = 202 "
 115                                 + " ORDER BY EcoFact.EcoFactId, PTaxon.RIdentifier, Fact.FactId ";
 116                 return result;
 117         }
 118
 119         /* (non-Javadoc)
 120          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
 121          */
 122         @Override
 123         protected String getRecordQuery(BerlinModelImportConfigurator config) {
 124                         String strQuery =   //DISTINCT because otherwise emOccurrenceSource creates multiple records for a single distribution
 125             " SELECT PTaxon.RIdentifier as taxonId, Fact.FactId, Fact.RecordBasis, EcoFact.*, " +
 126                " tg.ID AS GazetteerId, tg.L2Code, tg.L3Code, tg.L4Code, tg.Country, tg.ISOCountry, " +
 127                " ec.UUID as climateUuid, eh.UUID as habitatUuid, elf.UUID as lifeFormUuid" +
 128             " FROM Fact " +
 129                  " INNER JOIN EcoFact ON Fact.ExtensionFk = EcoFact.EcoFactId " +
 130                  " INNER JOIN PTaxon ON dbo.Fact.PTNameFk = dbo.PTaxon.PTNameFk AND dbo.Fact.PTRefFk = dbo.PTaxon.PTRefFk " +
 131                  " LEFT OUTER JOIN TDWGGazetteer tg ON EcoFact.TDWGGazetteerFk = tg.ID " +
 132                  " LEFT OUTER JOIN EcoClimate  ec  ON EcoFact.ClimateFk  = ec.ClimateId " +
 133                  " LEFT OUTER JOIN EcoHabitat  eh  ON EcoFact.HabitatFk  = eh.HabitatId " +
 134                  " LEFT OUTER JOIN EcoLifeForm elf ON EcoFact.LifeFormFk = elf.LifeFormId " +
 135               " WHERE Fact.FactCategoryFk = 202 AND (Fact.FactId IN (" + ID_LIST_TOKEN + ")  )"
 136             + " ORDER BY EcoFact.EcoFactId, PTaxon.RIdentifier, Fact.FactId "
 137             ;
 138                 return strQuery;
 139         }
 140
 141         /* (non-Javadoc)
 142          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 143          */
 144         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState bmState) {
 145                 boolean success = true;
 146
 147                 AlgaTerraImportState state = (AlgaTerraImportState)bmState;
 148                 try {
 149                         makeVocabulariesAndFeatures(state);
 150                 } catch (SQLException e1) {
 151                         logger.warn("Exception occurred when trying to create Ecofact vocabularies: " + e1.getMessage());
 152                         e1.printStackTrace();
 153                 }
 154                 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
 155
 156                 Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
 157                 Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>) partitioner.getObjectMap(FIELD_OBSERVATION_NAMESPACE);
 158
 159                 ResultSet rs = partitioner.getResultSet();
 160
 161                 try {
 162
 163                         int i = 0;
 164
 165                         //for each reference
 166             while (rs.next()){
 167
 168                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("Specimen facts handled: " + (i-1));}
 169
 170                                 int newTaxonId = rs.getInt("taxonId");
 171                                 int factId = rs.getInt("FactId");
 172                                 int ecoFactId = rs.getInt("EcoFactId");
 173                                 String recordBasis = rs.getString("RecordBasis");
 174
 175                                 try {
 176
 177                                         //source ref
 178                                         Reference<?> sourceRef = state.getTransactionalSourceReference();
 179
 180                                         //facade
 181                                         FieldObservation fieldObservation = getFieldObservation(ecoFactId, fieldObservationMap);
 182                                         DerivedUnitType type = makeDerivedUnitType(recordBasis);
 183                                         DerivedUnitFacade facade = DerivedUnitFacade.NewInstance(type, fieldObservation);
 184
 185                                         //field observation
 186                                         handleSingleSpecimen(rs, facade, state);
 187
 188                                         //description element
 189                                         TaxonDescription taxonDescription = getTaxonDescription(state, newTaxonId, taxonMap, factId, sourceRef);
 190                                         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
 191                                         Feature feature = makeFeature(type);
 192                                         indAssociation.setAssociatedSpecimenOrObservation(facade.innerDerivedUnit());
 193                                         indAssociation.setFeature(feature);
 194                                         taxonDescription.addElement(indAssociation);
 195
 196                                         taxaToSave.add(taxonDescription.getTaxon());
 197
 198
 199                                 } catch (Exception e) {
 200                                         logger.warn("Exception in ecoFact: FactId " + factId + ". " + e.getMessage());
 201 //                                      e.printStackTrace();
 202                                 }
 203
 204             }
 205
 206 //            logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
 207
 208                         logger.warn("Taxa to save: " + taxaToSave.size());
 209                         getTaxonService().save(taxaToSave);
 210
 211                         return success;
 212                 } catch (SQLException e) {
 213                         logger.error("SQLException:" +  e);
 214                         return false;
 215                 }
 216         }
 217
 218
 219
 220
 221         /**
 222          * Creates the vocabularies and the features for Climate, Habitat and Lifeform
 223          * @param state
 224          * @throws SQLException
 225          */
 226         private void makeVocabulariesAndFeatures(AlgaTerraImportState state) throws SQLException {
 227                 String abbrevLabel = null;
 228                 URI uri = null;
 229
 230                 if (! state.isSpecimenVocabulariesCreated()){
 231
 232                         TransactionStatus txStatus = this.startTransaction();
 233
 234                         boolean isOrdered = true;
 235                         OrderedTermVocabulary<State> climateVoc = (OrderedTermVocabulary)getVocabulary(uuidVocAlgaTerraClimate, "Climate", "Climate", abbrevLabel, uri, isOrdered, null);
 236                         OrderedTermVocabulary<State> habitatVoc = (OrderedTermVocabulary)getVocabulary(uuidVocAlgaTerraHabitat, "Habitat", "Habitat", abbrevLabel, uri, isOrdered, null);
 237                         OrderedTermVocabulary<State> lifeformVoc = (OrderedTermVocabulary)getVocabulary(uuidVocAlgaTerraLifeForm, "Lifeform", "Lifeform", abbrevLabel, uri, isOrdered, null);
 238
 239
 240                         Feature feature = getFeature(state, uuidFeatureAlgaTerraClimate, "Climate","Climate", null, null);
 241                         feature.setSupportsCategoricalData(true);
 242
 243                         feature = getFeature(state, uuidFeatureAlgaTerraLifeForm, "LifeForm","LifeForm", null, null);
 244                         feature.setSupportsCategoricalData(true);
 245
 246                         feature = Feature.HABITAT();
 247                         feature.setSupportsCategoricalData(true);
 248                         getTermService().saveOrUpdate(feature);
 249
 250                         Source source = state.getAlgaTerraConfigurator().getSource();
 251
 252                         String climateSql = "SELECT * FROM EcoClimate";
 253                         ResultSet rs = source.getResultSet(climateSql);
 254                         while (rs.next()){
 255                                 String climate = rs.getString("Climate");
 256                                 String description = rs.getString("Description");
 257                                 Integer id = rs.getInt("ClimateId");
 258                                 UUID uuid = UUID.fromString(rs.getString("UUID"));
 259                                 State stateTerm = getStateTerm(state, uuid, climate, description, null, climateVoc);
 260                                 addOriginalSource(stateTerm, id.toString(), "EcoClimate", state.getTransactionalSourceReference());
 261                                 getTermService().saveOrUpdate(stateTerm);
 262                         }
 263
 264                         String habitatSql = "SELECT * FROM EcoHabitat";
 265                         rs = source.getResultSet(habitatSql);
 266                         while (rs.next()){
 267                                 String habitat = rs.getString("Habitat");
 268                                 String description = rs.getString("Description");
 269                                 Integer id = rs.getInt("HabitatId");
 270                                 UUID uuid = UUID.fromString(rs.getString("UUID"));
 271                                 State stateTerm = getStateTerm(state, uuid, habitat, description, null, habitatVoc);
 272                                 addOriginalSource(stateTerm, id.toString(), "EcoHabitat", state.getTransactionalSourceReference());
 273                                 getTermService().saveOrUpdate(stateTerm);
 274                         }
 275
 276                         String lifeformSql = "SELECT * FROM EcoLifeForm";
 277                         rs = source.getResultSet(lifeformSql);
 278                         while (rs.next()){
 279                                 String lifeform = rs.getString("LifeForm");
 280                                 String description = rs.getString("Description");
 281                                 Integer id = rs.getInt("LifeFormId");
 282                                 UUID uuid = UUID.fromString(rs.getString("UUID"));
 283                                 State stateTerm = getStateTerm(state, uuid, lifeform, description, null, lifeformVoc);
 284                                 addOriginalSource(stateTerm, id.toString(), "EcoLifeForm", state.getTransactionalSourceReference());
 285                                 getTermService().saveOrUpdate(stateTerm);
 286                         }
 287
 288                         this.commitTransaction(txStatus);
 289
 290                         state.setSpecimenVocabulariesCreated(true);
 291                 }
 292
 293         }
 294
 295
 296
 297         private void handleSingleSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state) throws SQLException {
 298         //CollectionFk, Collector, AltitudeMethod,
 299         //ISOCountrySub, CreatedWhen/Who/Updated/who
 300
 301         //P1-10Value/Unit/Parameter/Method
 302
 303                 try {
 304                         Object alkalinityFlag = rs.getBoolean("AlkalinityFlag");
 305
 306                         String locality = rs.getString("Locality");
 307                         Double latitude = nullSafeDouble(rs, "Latitude");
 308                         Double longitude = nullSafeDouble(rs, "Longitude");
 309                         Integer errorRadius = nullSafeInt(rs,"Prec");
 310                         String geoCodeMethod = rs.getString("GeoCodeMethod");
 311
 312                         Integer altitude = nullSafeInt(rs, "Altitude");
 313                         Integer lowerAltitude = nullSafeInt(rs,"AltitudeLowerValue");
 314                         String altitudeUnit = rs.getString("AltitudeUnit");
 315                         Double depth = nullSafeDouble(rs, "Depth");
 316                         Double depthLow = nullSafeDouble(rs, "DepthLow");
 317
 318                         String collectorsNumber = rs.getString("CollectorsNumber");
 319                         Date collectionDateStart = rs.getDate("CollectionDate");
 320                         Date collectionDateEnd = rs.getDate("CollectionDateEnd");
 321
 322                         String climateUuid = rs.getString("climateUuid");
 323                         String habitatUuid = rs.getString("habitatUuid");
 324                         String lifeFormUuid = rs.getString("lifeFormUuid");
 325
 326                         String habitat = rs.getString("HabitatExplanation");
 327                         String community = rs.getString("Comunity");
 328                         String additionalData = rs.getString("AdditionalData");
 329
 330
 331
 332                         FieldObservation fieldObservation = facade.innerFieldObservation();
 333
 334                         //alkalinity marker
 335                         if (alkalinityFlag != null){
 336                                 MarkerType alkalinityMarkerType = getMarkerType(state, uuidMarkerAlkalinity, "Alkalinity", "Alkalinity", null);
 337                                 boolean alkFlag = Boolean.valueOf(alkalinityFlag.toString());
 338                                 Marker alkalinityMarker = Marker.NewInstance(alkalinityMarkerType, alkFlag);
 339                                 fieldObservation.addMarker(alkalinityMarker);
 340                         }
 341
 342                         //location
 343                         facade.setLocality(locality);
 344
 345                         //exact location
 346                         ReferenceSystem referenceSystem = makeRefrenceSystem(geoCodeMethod, state);
 347                         Point exactLocation = Point.NewInstance(longitude, latitude, referenceSystem, errorRadius);
 348                         facade.setExactLocation(exactLocation);
 349
 350                         //altitude, depth
 351                         if (StringUtils.isNotBlank(altitudeUnit) && ! altitudeUnit.trim().equalsIgnoreCase("m")){
 352                                 logger.warn("Altitude unit is not [m] but: " +  altitudeUnit);
 353                         }
 354                         if ( altitude != null){
 355                                 if (lowerAltitude == null){
 356                                         facade.setAbsoluteElevation(altitude);
 357                                 }else{
 358                                         if (! facade.isEvenDistance(lowerAltitude, altitude)){
 359                                                 //FIXME there is a ticket for this
 360                                                 altitude = altitude + 1;
 361                                                 logger.warn("Current implementation of altitude does not allow uneven distances");
 362                                         }
 363                                         facade.setAbsoluteElevationRange(lowerAltitude,altitude);
 364                                 }
 365                         }
 366                         if ( depth != null){
 367                                 //FIXME needs model change to accept double #3072
 368                                 Integer intDepth = depth.intValue();
 369                                 if (depthLow == null){
 370                                         facade.setDistanceToWaterSurface(intDepth);
 371                                 }else{
 372                                         //FIXME range not yet in model #3074
 373                                         facade.setDistanceToWaterSurface(intDepth);
 374                                 }
 375                         }
 376
 377                         //habitat, ecology, community, etc.
 378                         DescriptionBase<?> fieldDescription = getFieldObservationDescription(facade);
 379                         addCategoricalValue(state, fieldDescription, climateUuid, uuidFeatureAlgaTerraClimate);
 380                         addCategoricalValue(state, fieldDescription, habitatUuid, Feature.HABITAT().getUuid());
 381                         addCategoricalValue(state, fieldDescription, lifeFormUuid, uuidFeatureAlgaTerraLifeForm);
 382
 383                         if (isNotBlank(habitat)){
 384                                 //FIXME
 385                                 facade.setEcology(habitat);  //or use an own feature ??
 386                         }
 387                         if (isNotBlank(community)){
 388                                 Feature communityFeature = getFeature(state, uuidFeatureSpecimenCommunity, "Community", "The community of a specimen (e.g. other algae in the same sample)", null, null);
 389                                 TextData textData = TextData.NewInstance(communityFeature);
 390                                 textData.putText(Language.DEFAULT(), community);
 391                                 getFieldObservationDescription(facade).addElement(textData);
 392                         }
 393                         if (isNotBlank(additionalData)){  //or handle it as Annotation ??
 394                                 Feature additionalDataFeature = getFeature(state, uuidFeatureAdditionalData, "Additional Data", "Additional Data", null, null);
 395                                 TextData textData = TextData.NewInstance(additionalDataFeature);
 396                                 textData.putText(Language.DEFAULT(), additionalData);
 397                                 getFieldObservationDescription(facade).addElement(textData);
 398                         }
 399
 400                         //field
 401                         facade.setFieldNumber(collectorsNumber);
 402                         TimePeriod gatheringPeriod = TimePeriod.NewInstance(collectionDateStart, collectionDateEnd);
 403                         facade.setGatheringPeriod(gatheringPeriod);
 404                         handleCollectorTeam(state, facade, rs);
 405
 406                         //areas
 407                         makeAreas(state, rs, facade);
 408
 409                         //parameters
 410                         //TODO
 411
 412                         //collection
 413                         String voucher = rs.getString("Voucher");
 414                         if (StringUtils.isNotBlank(voucher)){
 415                                 facade.setAccessionNumber(voucher);
 416                         }
 417
 418
 419                         //notes
 420                         //TODO is this an annotation on field observation or on the derived unit?
 421
 422                         //TODO id, created for fact +  ecoFact
 423                         //      this.doIdCreatedUpdatedNotes(state, descriptionElement, rs, id, namespace);
 424
 425                 } catch (IllegalArgumentException e) {
 426                         throw e;
 427                 }
 428
 429         }
 430
 431
 432         private void addCategoricalValue(AlgaTerraImportState importState, DescriptionBase description, String uuidTerm, UUID featureUuid) {
 433                 if (uuidTerm != null){
 434                         State state = this.getStateTerm(importState, UUID.fromString(uuidTerm));
 435                         Feature feature = getFeature(importState, featureUuid);
 436                         CategoricalData categoricalData = CategoricalData.NewInstance(state, feature);
 437                         description.addElement(categoricalData);
 438                 }
 439         }
 440
 441
 442         private void handleCollectorTeam(AlgaTerraImportState state, DerivedUnitFacade facade, ResultSet rs) throws SQLException {
 443                 // FIXME parsen
 444                 String collector = rs.getString("Collector");
 445                 Team team = Team.NewTitledInstance(collector, collector);
 446                 facade.setCollector(team);
 447
 448
 449
 450         }
 451
 452         private void makeAreas(AlgaTerraImportState state, ResultSet rs, DerivedUnitFacade facade) throws SQLException {
 453                 Object gazetteerId = rs.getObject("GazetteerId");
 454                 if (gazetteerId != null){
 455                         //TDWG
 456                         NamedArea tdwgArea;
 457                         String tdwg4 = rs.getString("L4Code");
 458                         if (isNotBlank(tdwg4)){
 459                                 tdwgArea = TdwgArea.getAreaByTdwgAbbreviation(tdwg4);
 460                         }else{
 461                                 String tdwg3 = rs.getString("L3Code");
 462                                 if (isNotBlank(tdwg3)){
 463                                         tdwgArea = TdwgArea.getAreaByTdwgAbbreviation(tdwg3);
 464                                 }else{
 465                                         Integer tdwg2 = rs.getInt("L2Code");
 466                                         tdwgArea = TdwgArea.getAreaByTdwgAbbreviation(String.valueOf(tdwg2));
 467                                 }
 468                         }
 469                         if (tdwgArea == null){
 470                                 logger.warn("TDWG area could not be defined for gazetterId: " + gazetteerId);
 471                         }else{
 472                                 facade.addCollectingArea(tdwgArea);
 473                         }
 474
 475                         //Countries
 476                         WaterbodyOrCountry country = null;
 477                         String isoCountry = rs.getString("ISOCountry");
 478                         String countryStr = rs.getString("Country");
 479                         if (isNotBlank(isoCountry)){
 480                                 country = WaterbodyOrCountry.getWaterbodyOrCountryByIso3166A2(isoCountry);
 481                         }else if (isNotBlank(countryStr)){
 482                                 logger.warn("Country exists but no ISO code");
 483                         }
 484                         if (country == null){
 485                                 logger.warn("Country does not exist for GazetteerID " + gazetteerId);
 486                         }else{
 487                                 facade.setCountry(country);
 488                         }
 489
 490                 }
 491
 492                 //Waterbody
 493                 WaterbodyOrCountry waterbody = null;
 494                 String waterbodyStr = rs.getString("WaterBody");
 495                 if (isNotBlank(waterbodyStr)){
 496                         if (waterbodyStr.equals("Atlantic Ocean")){
 497                                 waterbody = WaterbodyOrCountry.ATLANTICOCEAN();
 498                         }else{
 499                                 logger.warn("Waterbody not recognized: " + waterbody);
 500                         }
 501                         if (waterbody != null){
 502                                 facade.addCollectingArea(waterbody);
 503                         }
 504                 }
 505
 506
 507                 //countries sub
 508                 //TODO
 509         }
 510
 511         private DescriptionBase getFieldObservationDescription(DerivedUnitFacade facade) {
 512                 Set<DescriptionBase> descriptions = facade.innerFieldObservation().getDescriptions();
 513                 for (DescriptionBase desc : descriptions){
 514                         if (desc.isImageGallery() == false){
 515                                 return desc;
 516                         }
 517                 }
 518                 SpecimenDescription specDesc = SpecimenDescription.NewInstance(facade.innerFieldObservation());
 519                 descriptions.add(specDesc);
 520                 return specDesc;
 521         }
 522
 523         private ReferenceSystem makeRefrenceSystem(String geoCodeMethod, AlgaTerraImportState state) {
 524                 if (StringUtils.isBlank(geoCodeMethod)){
 525                         return null;
 526                 }else if(geoCodeMethod.startsWith("GPS")){
 527                         getReferenceSystem(state, uuidRefSystemGps, "GPS", "GPS", "GPS", ReferenceSystem.GOOGLE_EARTH().getVocabulary());
 528                         return ReferenceSystem.WGS84();
 529                 }else if(geoCodeMethod.startsWith("Google")){
 530                         return ReferenceSystem.GOOGLE_EARTH();
 531                 }else if(geoCodeMethod.startsWith("Map")){
 532                         logger.warn("Reference system " +  geoCodeMethod +  " not yet supported.");
 533                         return null;
 534                 }else if(geoCodeMethod.startsWith("WikiProjekt Georeferenzierung") || geoCodeMethod.startsWith("http://toolserver.org/~geohack/geohack.php") ){
 535                         return ReferenceSystem.WGS84();
 536                 }else {
 537                         logger.warn("Reference system " +  geoCodeMethod +  " not yet supported.");
 538                         return null;
 539                 }
 540         }
 541
 542         private FieldObservation getFieldObservation(int ecoFactId, Map<String, FieldObservation> fieldObservationMap) {
 543                 String key = String.valueOf(ecoFactId);
 544                 FieldObservation fieldObservation = fieldObservationMap.get(key);
 545                 if (fieldObservation == null){
 546                         fieldObservation = FieldObservation.NewInstance();
 547
 548                         fieldObservationMap.put(key, fieldObservation);
 549                 }
 550
 551                 return fieldObservation;
 552         }
 553
 554         private Feature makeFeature(DerivedUnitType type) {
 555                 if (type.equals(DerivedUnitType.DerivedUnit)){
 556                         return Feature.INDIVIDUALS_ASSOCIATION();
 557                 }else if (type.equals(DerivedUnitType.FieldObservation) || type.equals(DerivedUnitType.Observation) ){
 558                         return Feature.OBSERVATION();
 559                 }else if (type.equals(DerivedUnitType.Fossil) || type.equals(DerivedUnitType.LivingBeing) || type.equals(DerivedUnitType.Specimen )){
 560                         return Feature.SPECIMEN();
 561                 }
 562                 logger.warn("No feature defined for derived unit type: " + type);
 563                 return null;
 564         }
 565
 566
 567         private DerivedUnitType makeDerivedUnitType(String recordBasis) {
 568                 DerivedUnitType result = null;
 569                 if (StringUtils.isBlank(recordBasis)){
 570                         result = DerivedUnitType.DerivedUnit;
 571                 } else if (recordBasis.equalsIgnoreCase("FossileSpecimen")){
 572                         result = DerivedUnitType.Fossil;
 573                 }else if (recordBasis.equalsIgnoreCase("HumanObservation")){
 574                         result = DerivedUnitType.Observation;
 575                 }else if (recordBasis.equalsIgnoreCase("Literature")){
 576                         logger.warn("Literature record basis not yet supported");
 577                         result = DerivedUnitType.DerivedUnit;
 578                 }else if (recordBasis.equalsIgnoreCase("LivingSpecimen")){
 579                         result = DerivedUnitType.LivingBeing;
 580                 }else if (recordBasis.equalsIgnoreCase("MachineObservation")){
 581                         logger.warn("MachineObservation record basis not yet supported");
 582                         result = DerivedUnitType.Observation;
 583                 }else if (recordBasis.equalsIgnoreCase("PreservedSpecimen")){
 584                         result = DerivedUnitType.Specimen;
 585                 }
 586                 return result;
 587         }
 588
 589         /* (non-Javadoc)
 590          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 591          */
 592         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
 593                 String nameSpace;
 594                 Class cdmClass;
 595                 Set<String> idSet;
 596                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 597
 598                 try{
 599                         Set<String> taxonIdSet = new HashSet<String>();
 600                         Set<String> fieldObservationIdSet = new HashSet<String>();
 601                         Set<String> termsIdSet = new HashSet<String>();
 602
 603                         while (rs.next()){
 604                                 handleForeignKey(rs, taxonIdSet, "taxonId");
 605                                 handleForeignKey(rs, fieldObservationIdSet, "ecoFactId");
 606                                 handleForeignKey(rs, termsIdSet, "ClimateFk");
 607                                 handleForeignKey(rs, termsIdSet, "HabitatFk");
 608                                 handleForeignKey(rs, termsIdSet, "LifeFormFk");
 609                         }
 610
 611                         //taxon map
 612                         nameSpace = BerlinModelTaxonImport.NAMESPACE;
 613                         cdmClass = TaxonBase.class;
 614                         idSet = taxonIdSet;
 615                         Map<String, TaxonBase> objectMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 616                         result.put(nameSpace, objectMap);
 617
 618                         //field observation map map
 619                         nameSpace = AlgaTerraSpecimenImport.FIELD_OBSERVATION_NAMESPACE;
 620                         cdmClass = FieldObservation.class;
 621                         idSet = taxonIdSet;
 622                         Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 623                         result.put(nameSpace, fieldObservationMap);
 624
 625                         //terms
 626                         nameSpace = AlgaTerraSpecimenImport.TERMS_NAMESPACE;
 627                         cdmClass = FieldObservation.class;
 628                         idSet = taxonIdSet;
 629                         Map<String, DefinedTermBase> termMap = (Map<String, DefinedTermBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 630                         result.put(nameSpace, termMap);
 631
 632
 633
 634
 635                 } catch (SQLException e) {
 636                         throw new RuntimeException(e);
 637                 }
 638                 return result;
 639         }
 640
 641
 642         /**
 643          * Use same TaxonDescription if two records belong to the same taxon
 644          * @param state
 645          * @param newTaxonId
 646          * @param oldTaxonId
 647          * @param oldDescription
 648          * @param taxonMap
 649          * @return
 650          */
 651         private TaxonDescription getTaxonDescription(AlgaTerraImportState state, int newTaxonId, Map<String, TaxonBase> taxonMap, int factId, Reference<?> sourceSec){
 652                 TaxonDescription result = null;
 653                 TaxonBase<?> taxonBase = taxonMap.get(String.valueOf(newTaxonId));
 654
 655                 //TODO for testing
 656                 if (taxonBase == null && ! state.getConfig().isDoTaxa()){
 657                         taxonBase = Taxon.NewInstance(BotanicalName.NewInstance(Rank.SPECIES()), null);
 658                 }
 659
 660                 Taxon taxon;
 661                 if ( taxonBase instanceof Taxon ) {
 662                         taxon = (Taxon) taxonBase;
 663                 } else if (taxonBase != null) {
 664                         logger.warn("TaxonBase for Fact(Specimen) with factId" + factId + " was not of type Taxon but: " + taxonBase.getClass().getSimpleName());
 665                         return null;
 666                 } else {
 667                         logger.warn("TaxonBase for Fact(Specimen) " + factId + " is null.");
 668                         return null;
 669                 }
 670                 Set<TaxonDescription> descriptionSet= taxon.getDescriptions();
 671                 if (descriptionSet.size() > 0) {
 672                         result = descriptionSet.iterator().next();
 673                 }else{
 674                         result = TaxonDescription.NewInstance();
 675                         result.setTitleCache(sourceSec.getTitleCache(), true);
 676                         taxon.addDescription(result);
 677                 }
 678                 return result;
 679         }
 680
 681
 682         /* (non-Javadoc)
 683          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 684          */
 685         @Override
 686         protected boolean doCheck(BerlinModelImportState state){
 687                 IOValidator<BerlinModelImportState> validator = new AlgaTerraSpecimenImportValidator();
 688                 return validator.validate(state);
 689         }
 690
 691         /* (non-Javadoc)
 692          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName()
 693          */
 694         @Override
 695         protected String getTableName() {
 696                 return dbTableName;
 697         }
 698
 699         /* (non-Javadoc)
 700          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString()
 701          */
 702         @Override
 703         public String getPluralString() {
 704                 return pluralString;
 705         }
 706
 707         /* (non-Javadoc)
 708          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 709          */
 710         protected boolean isIgnore(BerlinModelImportState state){
 711                 return ! ((AlgaTerraImportState)state).getAlgaTerraConfigurator().isDoSpecimen();
 712         }
 713
 714 }