app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraSpecimenImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.algaterra;
  11
  12 import java.sql.ResultSet;
  13 import java.sql.SQLException;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.Map;
  17 import java.util.Set;
  18 import java.util.UUID;
  19
  20 import org.apache.commons.lang.StringUtils;
  21 import org.apache.log4j.Logger;
  22 import org.springframework.stereotype.Component;
  23
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
  26 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeNotSupportedException;
  27 import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraSpecimenImportValidator;
  28 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
  29 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState;
  30 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonImport;
  31 import eu.etaxonomy.cdm.io.common.IOValidator;
  32 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  33 import eu.etaxonomy.cdm.model.common.CdmBase;
  34 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  35 import eu.etaxonomy.cdm.model.common.Language;
  36 import eu.etaxonomy.cdm.model.common.Marker;
  37 import eu.etaxonomy.cdm.model.common.MarkerType;
  38 import eu.etaxonomy.cdm.model.common.TermVocabulary;
  39 import eu.etaxonomy.cdm.model.description.CategoricalData;
  40 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  41 import eu.etaxonomy.cdm.model.description.Feature;
  42 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  43 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  44 import eu.etaxonomy.cdm.model.description.Modifier;
  45 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  46 import eu.etaxonomy.cdm.model.description.State;
  47 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  48 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  49 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  50 import eu.etaxonomy.cdm.model.description.TextData;
  51 import eu.etaxonomy.cdm.model.name.BotanicalName;
  52 import eu.etaxonomy.cdm.model.name.Rank;
  53 import eu.etaxonomy.cdm.model.occurrence.Collection;
  54 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  55 import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
  56 import eu.etaxonomy.cdm.model.reference.Reference;
  57 import eu.etaxonomy.cdm.model.taxon.Taxon;
  58 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  59
  60
  61 /**
  62  * @author a.mueller
  63  * @created 01.09.2012
  64  */
  65 @Component
  66 public class AlgaTerraSpecimenImport  extends AlgaTerraSpecimenImportBase {
  67         private static final Logger logger = Logger.getLogger(AlgaTerraSpecimenImport.class);
  68
  69
  70         private static int modCount = 5000;
  71         private static final String pluralString = "specimen and observation";
  72         private static final String dbTableName = "Fact";  //??
  73
  74
  75         public AlgaTerraSpecimenImport(){
  76                 super();
  77         }
  78
  79
  80
  81         /* (non-Javadoc)
  82          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery()
  83          */
  84         @Override
  85         protected String getIdQuery(BerlinModelImportState state) {
  86                 String result = " SELECT factId " +
  87                                 " FROM Fact " +
  88                                         " INNER JOIN EcoFact ON Fact.ExtensionFk = EcoFact.EcoFactId " +
  89                                         "INNER JOIN PTaxon ON Fact.PTNameFk = PTaxon.PTNameFk AND Fact.PTRefFk = PTaxon.PTRefFk "
  90                                 + " WHERE FactCategoryFk = 202 "
  91                                 + " ORDER BY EcoFact.EcoFactId, PTaxon.RIdentifier, Fact.FactId ";
  92                 return result;
  93         }
  94
  95         /* (non-Javadoc)
  96          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
  97          */
  98         @Override
  99         protected String getRecordQuery(BerlinModelImportConfigurator config) {
 100                         String strQuery =
 101             " SELECT PTaxon.RIdentifier as taxonId, Fact.FactId, Fact.RecordBasis, EcoFact.*, " +
 102                " tg.ID AS GazetteerId, tg.L2Code, tg.L3Code, tg.L4Code, tg.Country, tg.ISOCountry, " +
 103                " ec.UUID as climateUuid, eh.UUID as habitatUuid, elf.UUID as lifeFormUuid" +
 104             " FROM Fact " +
 105                  " INNER JOIN EcoFact ON Fact.ExtensionFk = EcoFact.EcoFactId " +
 106                  " INNER JOIN PTaxon ON dbo.Fact.PTNameFk = dbo.PTaxon.PTNameFk AND dbo.Fact.PTRefFk = dbo.PTaxon.PTRefFk " +
 107                  " LEFT OUTER JOIN TDWGGazetteer tg ON EcoFact.TDWGGazetteerFk = tg.ID " +
 108                  " LEFT OUTER JOIN EcoClimate  ec  ON EcoFact.ClimateFk  = ec.ClimateId " +
 109                  " LEFT OUTER JOIN EcoHabitat  eh  ON EcoFact.HabitatFk  = eh.HabitatId " +
 110                  " LEFT OUTER JOIN EcoLifeForm elf ON EcoFact.LifeFormFk = elf.LifeFormId " +
 111               " WHERE Fact.FactCategoryFk = 202 AND (Fact.FactId IN (" + ID_LIST_TOKEN + ")  )"
 112             + " ORDER BY EcoFact.EcoFactId, PTaxon.RIdentifier, Fact.FactId "
 113             ;
 114                 return strQuery;
 115         }
 116
 117         /* (non-Javadoc)
 118          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 119          */
 120         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState bmState) {
 121                 boolean success = true;
 122
 123                 AlgaTerraImportState state = (AlgaTerraImportState)bmState;
 124                 try {
 125                         makeVocabulariesAndFeatures(state);
 126                 } catch (SQLException e1) {
 127                         logger.warn("Exception occurred when trying to create Ecofact vocabularies: " + e1.getMessage());
 128                         e1.printStackTrace();
 129                 }
 130                 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
 131
 132                 Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
 133                 Map<String, DerivedUnit> ecoFactMap = (Map<String, DerivedUnit>) partitioner.getObjectMap(ECO_FACT_NAMESPACE);
 134
 135                 ResultSet rs = partitioner.getResultSet();
 136
 137                 try {
 138
 139                         int i = 0;
 140
 141                         //for each reference
 142             while (rs.next()){
 143
 144                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("Specimen facts handled: " + (i-1));}
 145
 146                                 int newTaxonId = rs.getInt("taxonId");
 147                                 int factId = rs.getInt("FactId");
 148                                 int ecoFactId = rs.getInt("EcoFactId");
 149                                 String recordBasis = rs.getString("RecordBasis");
 150
 151                                 try {
 152
 153                                         //source ref
 154                                         Reference<?> sourceRef = state.getTransactionalSourceReference();
 155
 156                                         //facade
 157                                         DerivedUnitType type = makeDerivedUnitType(recordBasis);
 158                                         DerivedUnitFacade facade = getDerivedUnit(state, ecoFactId, ecoFactMap, type);
 159
 160                                         //field observation
 161                                         handleSingleSpecimen(rs, facade, state, partitioner);
 162
 163                                         handleEcoFactSpecificSpecimen(rs,facade, state);
 164
 165                                         state.setCurrentFieldObservationNotNew(false);
 166
 167                                         //description element
 168                                         TaxonDescription taxonDescription = getTaxonDescription(state, newTaxonId, taxonMap, factId, sourceRef);
 169                                         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
 170                                         Feature feature = makeFeature(type);
 171                                         indAssociation.setAssociatedSpecimenOrObservation(facade.innerDerivedUnit());
 172                                         indAssociation.setFeature(feature);
 173                                         taxonDescription.addElement(indAssociation);
 174
 175                                         taxaToSave.add(taxonDescription.getTaxon());
 176
 177
 178                                 } catch (Exception e) {
 179                                         logger.warn("Exception in ecoFact: FactId " + factId + ". " + e.getMessage());
 180 //                                      e.printStackTrace();
 181                                 }
 182
 183             }
 184
 185 //            logger.warn("Specimen: " + countSpecimen + ", Descriptions: " + countDescriptions );
 186
 187                         logger.warn("Taxa to save: " + taxaToSave.size());
 188                         getTaxonService().save(taxaToSave);
 189
 190                         return success;
 191                 } catch (SQLException e) {
 192                         logger.error("SQLException:" +  e);
 193                         return false;
 194                 }
 195         }
 196
 197
 198
 199         private void handleEcoFactSpecificSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state) throws SQLException {
 200
 201                 Object alkalinityFlag = rs.getBoolean("AlkalinityFlag");
 202
 203                 //alkalinity marker
 204                 if (alkalinityFlag != null){
 205                         MarkerType alkalinityMarkerType = getMarkerType(state, uuidMarkerAlkalinity, "Alkalinity", "Alkalinity", null);
 206                         boolean alkFlag = Boolean.valueOf(alkalinityFlag.toString());
 207                         Marker alkalinityMarker = Marker.NewInstance(alkalinityMarkerType, alkFlag);
 208                         facade.getFieldObservation(true).addMarker(alkalinityMarker);
 209                 }
 210
 211
 212                 DescriptionBase<?> fieldDescription = getFieldObservationDescription(facade);
 213
 214                 //habitat, ecology, community, etc.
 215                 String habitat = rs.getString("HabitatExplanation");
 216
 217                 if (isNotBlank(habitat)){
 218                         Feature habitatExplanation = getFeature(state, uuidFeatureHabitatExplanation, "Habitat Explanation", "HabitatExplanation", null, null);
 219                         TextData textData = TextData.NewInstance(habitatExplanation);
 220                         textData.putText(Language.DEFAULT(), habitat);
 221                         fieldDescription.addElement(textData);
 222                 }
 223
 224                 String community = rs.getString("Comunity");
 225                 if (isNotBlank(community)){
 226                         Feature communityFeature = getFeature(state, uuidFeatureSpecimenCommunity, "Community", "The community of a specimen (e.g. other algae in the same sample)", null, null);
 227                         TextData textData = TextData.NewInstance(communityFeature);
 228                         textData.putText(Language.DEFAULT(), community);
 229                         fieldDescription.addElement(textData);
 230                 }
 231
 232                 String additionalData = rs.getString("AdditionalData");
 233                 if (isNotBlank(additionalData)){  //or handle it as Annotation ??
 234                         Feature additionalDataFeature = getFeature(state, uuidFeatureAdditionalData, "Additional Data", "Additional Data", null, null);
 235                         TextData textData = TextData.NewInstance(additionalDataFeature);
 236                         textData.putText(Language.DEFAULT(), additionalData);
 237                         fieldDescription.addElement(textData);
 238                 }
 239
 240                 String climateUuid = rs.getString("climateUuid");
 241                 String habitatUuid = rs.getString("habitatUuid");
 242                 String lifeFormUuid = rs.getString("lifeFormUuid");
 243
 244                 addCategoricalValue(state, fieldDescription, climateUuid, uuidFeatureAlgaTerraClimate);
 245                 addCategoricalValue(state, fieldDescription, habitatUuid, Feature.HABITAT().getUuid());
 246                 addCategoricalValue(state, fieldDescription, lifeFormUuid, uuidFeatureAlgaTerraLifeForm);
 247
 248
 249                 //collection
 250                 String voucher = rs.getString("Voucher");
 251                 if (StringUtils.isNotBlank(voucher)){
 252                         facade.setAccessionNumber(voucher);
 253                 }
 254
 255                 //parameters
 256                 makeParameter(state, rs, getFieldObservationDescription(facade));
 257
 258         }
 259
 260
 261
 262
 263         private void addCategoricalValue(AlgaTerraImportState importState, DescriptionBase description, String uuidTerm, UUID featureUuid) {
 264                 if (uuidTerm != null){
 265                         State state = this.getStateTerm(importState, UUID.fromString(uuidTerm));
 266                         Feature feature = getFeature(importState, featureUuid);
 267                         CategoricalData categoricalData = CategoricalData.NewInstance(state, feature);
 268                         description.addElement(categoricalData);
 269                 }
 270         }
 271
 272         private void makeParameter(AlgaTerraImportState state, ResultSet rs, DescriptionBase<?> descriptionBase) throws SQLException {
 273                 for (int i = 1; i <= 10; i++){
 274                         String valueStr = rs.getString(String.format("P%dValue", i));
 275                         String unitStr = rs.getString(String.format("P%dUnit", i));
 276                         String parameter = rs.getString(String.format("P%dParameter", i));
 277                         String method = rs.getString(String.format("P%dMethod", i));
 278
 279                         //method
 280                         if (StringUtils.isNotBlank(method)){
 281                                 logger.warn("Methods not yet handled: " + method);
 282                         }
 283                         //parameter
 284                         TermVocabulary<Feature> vocParameter = getVocabulary(uuidVocParameter, "Feature vocabulary for AlgaTerra measurement parameters", "Parameters", null, null, false, Feature.COMMON_NAME());
 285                         if (StringUtils.isNotBlank(parameter)){
 286                                 UUID featureUuid = getParameterFeatureUuid(state, parameter);
 287                                 Feature feature = getFeature(state, featureUuid, parameter, parameter, null, vocParameter);
 288                                 QuantitativeData quantData = QuantitativeData.NewInstance(feature);
 289
 290                                 //unit
 291                                 MeasurementUnit unit = getMeasurementUnit(state, unitStr);
 292                                 quantData.setUnit(unit);
 293                                 try {
 294
 295                                         Set<Modifier> valueModifier = new HashSet<Modifier>();
 296                                         valueStr = normalizeAndModifyValue(state, valueStr, valueModifier);
 297                                         //value
 298                                         Float valueFlt = Float.valueOf(valueStr);  //TODO maybe change model to Double ??
 299
 300                                         StatisticalMeasure measureSingleValue = getStatisticalMeasure(state, uuidStatMeasureSingleValue, "Value", "Single measurement value", null, null);
 301                                         StatisticalMeasurementValue value = StatisticalMeasurementValue.NewInstance(measureSingleValue, valueFlt);
 302                                         quantData.addStatisticalValue(value);
 303                                         descriptionBase.addElement(quantData);
 304
 305                                 } catch (NumberFormatException e) {
 306                                         logger.warn(String.format("Value '%s' can't be converted to double. Parameter %s not imported.", valueStr, parameter));
 307                                 }
 308                         }else if (isNotBlank(valueStr) || isNotBlank(unitStr) ){
 309                                 logger.warn("There is value or unit without parameter: " + i);
 310                         }
 311
 312
 313                 }
 314
 315         }
 316
 317         private String normalizeAndModifyValue(AlgaTerraImportState state, String valueStr, Set<Modifier> valueModifier) {
 318                 valueStr = valueStr.replace(",", ".");
 319                 if (valueStr.startsWith("<")){
 320                         TermVocabulary<Modifier> measurementValueModifierVocabulary = getVocabulary(uuidMeasurementValueModifier, "Measurement value modifier", "Measurement value modifier", null, null, false, Modifier.NewInstance());
 321                         Modifier modifier = getModifier(state, uuidModifierLowerThan, "Lower", "Lower than the given measurement value", "<", measurementValueModifierVocabulary);
 322                         valueModifier.add(modifier);
 323                         valueStr = valueStr.replace("<", "");
 324                 }
 325                 if (valueStr.startsWith(">")){
 326                         TermVocabulary<Modifier> measurementValueModifierVocabulary = getVocabulary(uuidMeasurementValueModifier, "Measurement value modifier", "Measurement value modifier", null, null, false, Modifier.NewInstance());
 327                         Modifier modifier = getModifier(state, uuidModifierGreaterThan, "Lower", "Lower than the given measurement value", "<", measurementValueModifierVocabulary);
 328                         valueModifier.add(modifier);
 329                         valueStr = valueStr.replace(">", "");
 330                 }
 331                 return valueStr;
 332         }
 333
 334
 335
 336         private UUID getParameterFeatureUuid(AlgaTerraImportState state, String key) {
 337                 //TODO define some UUIDs in Transformer
 338                 UUID uuid = state.getParameterFeatureUuid(key);
 339                 if (uuid == null){
 340                         uuid = UUID.randomUUID();
 341                         state.putParameterFeatureUuid(key, uuid);
 342                 }
 343                 return uuid;
 344         }
 345
 346
 347
 348         /**
 349          * TODO move to InputTransformerBase
 350          * @param state
 351          * @param unitStr
 352          * @return
 353          */
 354         private MeasurementUnit getMeasurementUnit(AlgaTerraImportState state, String unitStr) {
 355                 if (StringUtils.isNotBlank(unitStr)){
 356                         UUID uuidMeasurementUnitMgL = UUID.fromString("7ac302c5-3cbd-4334-964a-bf5d11eb9ead");
 357                         UUID uuidMeasurementUnitMolMol = UUID.fromString("96b78d78-3e49-448f-8100-e7779b71dd53");
 358                         UUID uuidMeasurementUnitMicroMolSiL = UUID.fromString("2cb8bc85-a4af-42f1-b80b-34c36c9f75d4");
 359                         UUID uuidMeasurementUnitMicroMolL = UUID.fromString("a631f62e-377e-405c-bd1a-76885b13a72b");
 360                         UUID uuidMeasurementUnitDegreeC = UUID.fromString("55222aec-d5be-413e-8db7-d9a48c316c6c");
 361                         UUID uuidMeasurementUnitPercent = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
 362                         UUID uuidMeasurementUnitCm = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
 363                         UUID uuidMeasurementUnitMicroSiCm = UUID.fromString("3ea3110e-f048-4bed-8bfe-33c60f63626f");
 364
 365
 366                         if (unitStr.equalsIgnoreCase("mg/L")){
 367                                 return getMeasurementUnit(state, uuidMeasurementUnitMgL, unitStr, unitStr, unitStr, null);
 368                         }else if (unitStr.equalsIgnoreCase("mol/mol")){
 369                                 return getMeasurementUnit(state, uuidMeasurementUnitMolMol, unitStr, unitStr, unitStr, null);
 370                         }else if (unitStr.equalsIgnoreCase("\u00B5mol Si/L")){   //µmol Si/L
 371                                 return getMeasurementUnit(state, uuidMeasurementUnitMicroMolSiL, unitStr, unitStr, unitStr, null);
 372                         }else if (unitStr.equalsIgnoreCase("\u00B5mol/L")){             //µmol/L
 373                                 return getMeasurementUnit(state, uuidMeasurementUnitMicroMolL, unitStr, unitStr, unitStr, null);
 374                         }else if (unitStr.equalsIgnoreCase("\u00B0C")){               //°C
 375                                 return getMeasurementUnit(state, uuidMeasurementUnitDegreeC, unitStr, unitStr, unitStr, null);
 376                         }else if (unitStr.equalsIgnoreCase("%")){
 377                                 return getMeasurementUnit(state, uuidMeasurementUnitPercent, unitStr, unitStr, unitStr, null);
 378                         }else if (unitStr.equalsIgnoreCase("cm")){
 379                                 return getMeasurementUnit(state, uuidMeasurementUnitCm, unitStr, unitStr, unitStr, null);
 380                         }else if (unitStr.equalsIgnoreCase("\u00B5S/cm")){   //µS/cm
 381                                 return getMeasurementUnit(state, uuidMeasurementUnitMicroSiCm, unitStr, unitStr, unitStr, null);
 382                         }else{
 383                                 logger.warn("MeasurementUnit was not recognized");
 384                                 return null;
 385                         }
 386                 }else{
 387                         return null;
 388                 }
 389         }
 390
 391
 392
 393
 394
 395         /**
 396          * @param state
 397          * @param ecoFactId
 398          * @param derivedUnitMap
 399          * @param type
 400          * @return
 401          */
 402         private DerivedUnitFacade getDerivedUnit(AlgaTerraImportState state, int ecoFactId, Map<String, DerivedUnit> derivedUnitMap, DerivedUnitType type) {
 403                 String key = String.valueOf(ecoFactId);
 404                 DerivedUnit derivedUnit = derivedUnitMap.get(key);
 405                 DerivedUnitFacade facade;
 406                 if (derivedUnit == null){
 407                         facade = DerivedUnitFacade.NewInstance(type);
 408                         derivedUnitMap.put(key, derivedUnit);
 409                 }else{
 410                         try {
 411                                 facade = DerivedUnitFacade.NewInstance(derivedUnit);
 412                         } catch (DerivedUnitFacadeNotSupportedException e) {
 413                                 logger.error(e.getMessage());
 414                                 facade = DerivedUnitFacade.NewInstance(type);
 415                         }
 416                 }
 417
 418                 return facade;
 419         }
 420
 421         private Feature makeFeature(DerivedUnitType type) {
 422                 if (type.equals(DerivedUnitType.DerivedUnit)){
 423                         return Feature.INDIVIDUALS_ASSOCIATION();
 424                 }else if (type.equals(DerivedUnitType.FieldObservation) || type.equals(DerivedUnitType.Observation) ){
 425                         return Feature.OBSERVATION();
 426                 }else if (type.equals(DerivedUnitType.Fossil) || type.equals(DerivedUnitType.LivingBeing) || type.equals(DerivedUnitType.Specimen )){
 427                         return Feature.SPECIMEN();
 428                 }
 429                 logger.warn("No feature defined for derived unit type: " + type);
 430                 return null;
 431         }
 432
 433
 434         private DerivedUnitType makeDerivedUnitType(String recordBasis) {
 435                 DerivedUnitType result = null;
 436                 if (StringUtils.isBlank(recordBasis)){
 437                         result = DerivedUnitType.DerivedUnit;
 438                 } else if (recordBasis.equalsIgnoreCase("FossileSpecimen")){
 439                         result = DerivedUnitType.Fossil;
 440                 }else if (recordBasis.equalsIgnoreCase("HumanObservation")){
 441                         result = DerivedUnitType.Observation;
 442                 }else if (recordBasis.equalsIgnoreCase("Literature")){
 443                         logger.warn("Literature record basis not yet supported");
 444                         result = DerivedUnitType.DerivedUnit;
 445                 }else if (recordBasis.equalsIgnoreCase("LivingSpecimen")){
 446                         result = DerivedUnitType.LivingBeing;
 447                 }else if (recordBasis.equalsIgnoreCase("MachineObservation")){
 448                         logger.warn("MachineObservation record basis not yet supported");
 449                         result = DerivedUnitType.Observation;
 450                 }else if (recordBasis.equalsIgnoreCase("PreservedSpecimen")){
 451                         result = DerivedUnitType.Specimen;
 452                 }
 453                 return result;
 454         }
 455
 456         /* (non-Javadoc)
 457          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 458          */
 459         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
 460                 String nameSpace;
 461                 Class cdmClass;
 462                 Set<String> idSet;
 463                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 464
 465                 try{
 466                         Set<String> taxonIdSet = new HashSet<String>();
 467                         Set<String> fieldObservationIdSet = new HashSet<String>();
 468                         Set<String> termsIdSet = new HashSet<String>();
 469                         Set<String> collectionIdSet = new HashSet<String>();
 470
 471                         while (rs.next()){
 472                                 handleForeignKey(rs, taxonIdSet, "taxonId");
 473                                 handleForeignKey(rs, fieldObservationIdSet, "ecoFactId");
 474                                 handleForeignKey(rs, termsIdSet, "ClimateFk");
 475                                 handleForeignKey(rs, termsIdSet, "HabitatFk");
 476                                 handleForeignKey(rs, termsIdSet, "LifeFormFk");
 477                                 handleForeignKey(rs, collectionIdSet, "CollectionFk");
 478                         }
 479
 480                         //taxon map
 481                         nameSpace = BerlinModelTaxonImport.NAMESPACE;
 482                         cdmClass = TaxonBase.class;
 483                         idSet = taxonIdSet;
 484                         Map<String, TaxonBase> objectMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 485                         result.put(nameSpace, objectMap);
 486
 487                         //field observation map map
 488                         nameSpace = AlgaTerraSpecimenImport.ECO_FACT_NAMESPACE;
 489                         cdmClass = FieldObservation.class;
 490                         idSet = fieldObservationIdSet;
 491                         Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 492                         result.put(nameSpace, fieldObservationMap);
 493
 494                         //collections
 495                         nameSpace = AlgaTerraCollectionImport.NAMESPACE_COLLECTION;
 496                         cdmClass = Collection.class;
 497                         idSet = collectionIdSet;
 498                         Map<String, Collection> collectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 499                         result.put(nameSpace, collectionMap);
 500
 501                         //sub-collections
 502                         nameSpace = AlgaTerraCollectionImport.NAMESPACE_SUBCOLLECTION;
 503                         cdmClass = Collection.class;
 504                         idSet = collectionIdSet;
 505                         Map<String, Collection> subCollectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 506                         result.put(nameSpace, subCollectionMap);
 507
 508                         //terms
 509                         nameSpace = AlgaTerraSpecimenImport.TERMS_NAMESPACE;
 510                         cdmClass = FieldObservation.class;
 511                         idSet = termsIdSet;
 512                         Map<String, DefinedTermBase> termMap = (Map<String, DefinedTermBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 513                         result.put(nameSpace, termMap);
 514
 515                 } catch (SQLException e) {
 516                         throw new RuntimeException(e);
 517                 }
 518                 return result;
 519         }
 520
 521
 522         /**
 523          * Use same TaxonDescription if two records belong to the same taxon
 524          * @param state
 525          * @param newTaxonId
 526          * @param oldTaxonId
 527          * @param oldDescription
 528          * @param taxonMap
 529          * @return
 530          */
 531         private TaxonDescription getTaxonDescription(AlgaTerraImportState state, int newTaxonId, Map<String, TaxonBase> taxonMap, int factId, Reference<?> sourceSec){
 532                 TaxonDescription result = null;
 533                 TaxonBase<?> taxonBase = taxonMap.get(String.valueOf(newTaxonId));
 534
 535                 //TODO for testing
 536                 if (taxonBase == null && ! state.getConfig().isDoTaxa()){
 537                         taxonBase = Taxon.NewInstance(BotanicalName.NewInstance(Rank.SPECIES()), null);
 538                 }
 539
 540                 Taxon taxon;
 541                 if ( taxonBase instanceof Taxon ) {
 542                         taxon = (Taxon) taxonBase;
 543                 } else if (taxonBase != null) {
 544                         logger.warn("TaxonBase for Fact(Specimen) with factId" + factId + " was not of type Taxon but: " + taxonBase.getClass().getSimpleName());
 545                         return null;
 546                 } else {
 547                         logger.warn("TaxonBase for Fact(Specimen) " + factId + " is null.");
 548                         return null;
 549                 }
 550                 Set<TaxonDescription> descriptionSet= taxon.getDescriptions();
 551                 if (descriptionSet.size() > 0) {
 552                         result = descriptionSet.iterator().next();
 553                 }else{
 554                         result = TaxonDescription.NewInstance();
 555                         result.setTitleCache(sourceSec.getTitleCache(), true);
 556                         taxon.addDescription(result);
 557                 }
 558                 return result;
 559         }
 560
 561
 562         /* (non-Javadoc)
 563          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 564          */
 565         @Override
 566         protected boolean doCheck(BerlinModelImportState state){
 567                 IOValidator<BerlinModelImportState> validator = new AlgaTerraSpecimenImportValidator();
 568                 return validator.validate(state);
 569         }
 570
 571         /* (non-Javadoc)
 572          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName()
 573          */
 574         @Override
 575         protected String getTableName() {
 576                 return dbTableName;
 577         }
 578
 579         /* (non-Javadoc)
 580          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString()
 581          */
 582         @Override
 583         public String getPluralString() {
 584                 return pluralString;
 585         }
 586
 587         /* (non-Javadoc)
 588          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 589          */
 590         protected boolean isIgnore(BerlinModelImportState state){
 591                 return ! ((AlgaTerraImportState)state).getAlgaTerraConfigurator().isDoSpecimen();
 592         }
 593
 594 }