app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraDnaImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.algaterra;
  11
  12 import java.net.URI;
  13 import java.sql.ResultSet;
  14 import java.sql.SQLException;
  15 import java.text.ParseException;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.Locale;
  19 import java.util.Map;
  20 import java.util.Set;
  21
  22 import org.apache.commons.lang.StringUtils;
  23 import org.apache.log4j.Logger;
  24 import org.joda.time.DateTime;
  25 import org.joda.time.format.DateTimeFormat;
  26 import org.joda.time.format.DateTimeFormatter;
  27 import org.springframework.format.datetime.joda.DateTimeParser;
  28 import org.springframework.stereotype.Component;
  29
  30 import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraDnaImportValidator;
  31 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
  32 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState;
  33 import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonImport;
  34 import eu.etaxonomy.cdm.io.common.IOValidator;
  35 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  36 import eu.etaxonomy.cdm.model.common.Annotation;
  37 import eu.etaxonomy.cdm.model.common.AnnotationType;
  38 import eu.etaxonomy.cdm.model.common.CdmBase;
  39 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  40 import eu.etaxonomy.cdm.model.common.Language;
  41 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  42 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  43 import eu.etaxonomy.cdm.model.molecular.DnaSample;
  44 import eu.etaxonomy.cdm.model.molecular.Sequence;
  45 import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
  46 import eu.etaxonomy.cdm.model.occurrence.DerivationEventType;
  47 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  48 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  49 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  50 import eu.etaxonomy.cdm.model.reference.Reference;
  51 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  52 import eu.etaxonomy.cdm.model.taxon.Taxon;
  53 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  54
  55
  56 /**
  57  * @author a.mueller
  58  * @created 01.09.2012
  59  */
  60 @Component
  61 public class AlgaTerraDnaImport  extends AlgaTerraSpecimenImportBase {
  62         private static final Logger logger = Logger.getLogger(AlgaTerraDnaImport.class);
  63
  64
  65         private static int modCount = 5000;
  66         private static final String pluralString = "dna facts";
  67         private static final String dbTableName = "DNAFact";  //??
  68
  69
  70         public AlgaTerraDnaImport(){
  71                 super(dbTableName, pluralString);
  72         }
  73
  74
  75
  76         /* (non-Javadoc)
  77          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery()
  78          */
  79         @Override
  80         protected String getIdQuery(BerlinModelImportState bmState) {
  81                 AlgaTerraImportState state = (AlgaTerraImportState)bmState;
  82                 String result = " SELECT df.DNAFactId " +
  83                                 " FROM DNAFact df " +
  84                                         " INNER JOIN Fact f ON  f.ExtensionFk = df.DNAFactID " +
  85                                         " WHERE f.FactCategoryFk = 203 ";
  86                 if (state.getAlgaTerraConfigurator().isRemoveRestricted()){
  87                                 result = result + " AND df.ProtectedFlag = 0 ";
  88                 }
  89                 result += " ORDER BY df.DNAFactID ";
  90                 return result;
  91         }
  92
  93         /* (non-Javadoc)
  94          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
  95          */
  96         @Override
  97         protected String getRecordQuery(BerlinModelImportConfigurator config) {
  98                         String strQuery =
  99                     " SELECT df.*, pt.RIdentifier as taxonId, f.FactId, f.restrictedFlag, ecoFact.ecoFactId as ecoFactId " +
 100                     " FROM DNAFact df INNER JOIN Fact f ON  f.ExtensionFk = df.DNAFactID " +
 101                         " LEFT OUTER JOIN PTaxon pt ON f.PTNameFk = pt.PTNameFk AND f.PTRefFk = pt.PTRefFk " +
 102                         " LEFT OUTER JOIN EcoFact ecoFact ON ecoFact.CultureStrain = df.CultureStrainNo " +
 103                       " WHERE f.FactCategoryFk = 203 AND (df.DNAFactId IN (" + ID_LIST_TOKEN + ")  )"
 104                     + " ORDER BY DNAFactID "
 105             ;
 106                 return strQuery;
 107         }
 108
 109         @Override
 110         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState bmState) {
 111                 boolean success = true;
 112
 113                 AlgaTerraImportState state = (AlgaTerraImportState)bmState;
 114                 try {
 115 //                      makeVocabulariesAndFeatures(state);
 116                 } catch (Exception e1) {
 117                         logger.warn("Exception occurred when trying to create Ecofact vocabularies: " + e1.getMessage());
 118                         e1.printStackTrace();
 119                 }
 120                 Set<SpecimenOrObservationBase> samplesToSave = new HashSet<SpecimenOrObservationBase>();
 121                 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
 122
 123                 Map<String, FieldUnit> ecoFactFieldObservationMap = (Map<String, FieldUnit>) partitioner.getObjectMap(ECO_FACT_FIELD_OBSERVATION_NAMESPACE);
 124
 125                 ResultSet rs = partitioner.getResultSet();
 126
 127                 Map<String, Reference> referenceMap = new HashMap<String, Reference>();
 128
 129
 130                 try {
 131
 132                         int i = 0;
 133
 134                         //for each reference
 135             while (rs.next()){
 136
 137                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info(pluralString + " handled: " + (i-1));}
 138
 139                                 int dnaFactId = rs.getInt("DNAFactId");
 140                                 String keywordsStr = rs.getString("Keywords");
 141                                 String locusStr = rs.getString("Locus");
 142                                 String definitionStr = rs.getString("Definition");
 143
 144
 145                                 try {
 146
 147                                         //source ref
 148                                         Reference<?> sourceRef = state.getTransactionalSourceReference();
 149
 150                                         //import date
 151                                         DateTime importDateTime = makeImportDateTime(rs);
 152
 153                                         //DNA Sample
 154                                         DnaSample dnaSample = DnaSample.NewInstance();
 155                                         dnaSample.setCreated(importDateTime);
 156
 157                                         //ecoFactFk
 158                                         makeDerivationFromEcoFact(state, rs, dnaSample, samplesToSave, dnaFactId);
 159
 160                                         //sequence
 161                                         Sequence sequence = makeSequence(rs, dnaSample, dnaFactId, importDateTime);
 162
 163                                         //locus
 164                                         //FIXME Deduplicate DnaMarker
 165                                         DefinedTerm locus = DefinedTerm.NewDnaMarkerInstance(definitionStr, keywordsStr, null);
 166                                         locus.setCreated(importDateTime);
 167                                         this.getTermService().save(locus);
 168
 169                                         sequence.setDnaMarker(locus);
 170
 171                                         //GenBank Accession
 172                                         makeGenBankAccession(rs, sequence, importDateTime, dnaFactId);
 173
 174                                         //Comment
 175                                         String commentStr = rs.getString("Comment");
 176                                         if (isNotBlank(commentStr)){
 177                                                 Annotation annotation = Annotation.NewInstance(commentStr, AnnotationType.EDITORIAL(), Language.DEFAULT());
 178                                                 annotation.setCreated(importDateTime);
 179                                                 sequence.addAnnotation(annotation);
 180                                         }
 181
 182                                         //Indiv.Assoc.
 183                                         makeIndividualsAssociation(partitioner, rs, state, taxaToSave, dnaSample);
 184
 185                                         //TODO titleCache
 186                                         //prelim implementation:
 187                                         String cultStrain = rs.getString("CultureStrainNo");
 188                                         String title = String.format("DNA Sample for %s at %s", cultStrain, keywordsStr);
 189                                         dnaSample.setTitleCache(title, true);
 190
 191                                         //TODO preliminary implementation
 192                                         String referenceStr = rs.getString("FactReference");
 193                                         if (isNotBlank(referenceStr)){
 194                                                 Reference<?> ref = referenceMap.get(referenceStr);
 195                                                 if (ref == null){
 196                                                         ref = ReferenceFactory.newGeneric();
 197                                                         ref.setTitleCache(referenceStr, true);
 198                                                         referenceMap.put(referenceStr, ref);
 199                                                 }
 200                                                 sequence.addCitation(ref);
 201                                         }
 202
 203                                         //save
 204                                         samplesToSave.add(dnaSample);
 205
 206
 207                                 } catch (Exception e) {
 208                                         logger.warn("Exception in ecoFact: ecoFactId " + dnaFactId + ". " + e.getMessage());
 209                                         e.printStackTrace();
 210                                 }
 211
 212             }
 213
 214                         logger.warn("DNASample or EcoFacts to save: " + samplesToSave.size());
 215                         getOccurrenceService().saveOrUpdate(samplesToSave);
 216                         logger.warn("Taxa to save: " + samplesToSave.size());
 217                         getTaxonService().saveOrUpdate(taxaToSave);
 218
 219                         return success;
 220                 } catch (SQLException e) {
 221                         logger.error("SQLException:" +  e);
 222                         return false;
 223                 }
 224         }
 225
 226
 227         private void makeDerivationFromEcoFact(AlgaTerraImportState state, ResultSet rs, DnaSample dnaSample, Set<SpecimenOrObservationBase> samplesToSave, Integer dnaFactId) throws SQLException {
 228                 Integer ecoFactFk = nullSafeInt(rs, "ecoFactId");
 229                 if (ecoFactFk != null){
 230
 231                         DerivedUnit ecoFact = (DerivedUnit)state.getRelatedObject(ECO_FACT_DERIVED_UNIT_NAMESPACE, ecoFactFk.toString());
 232                         if (ecoFact == null){
 233                                 logger.warn("EcoFact is null for ecoFactFk: " + ecoFactFk + ", DnaFactId: " + dnaFactId);
 234                         }else{
 235                                 DerivationEvent.NewSimpleInstance(ecoFact, dnaSample, DerivationEventType.DNA_EXTRACTION());
 236                                 samplesToSave.add(ecoFact);
 237                         }
 238                 }
 239         }
 240
 241
 242
 243         private void makeIndividualsAssociation(ResultSetPartitioner partitioner, ResultSet rs, AlgaTerraImportState state, Set<TaxonBase> taxaToSave, DnaSample dnaSample) throws SQLException{
 244                 Reference<?> sourceRef = state.getTransactionalSourceReference();
 245                 Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
 246                 Integer taxonId = rs.getInt("taxonId");
 247                 Integer factId = rs.getInt("factId");
 248                 Taxon taxon = getTaxon(state, taxonId, taxonMap, factId);
 249                 TaxonDescription desc = getTaxonDescription(state, taxon, sourceRef);
 250                 IndividualsAssociation assoc = IndividualsAssociation.NewInstance(dnaSample);
 251                 desc.addElement(assoc);
 252                 taxaToSave.add(taxon);
 253         }
 254
 255
 256         /**
 257          * @param rs
 258          * @return
 259          * @throws SQLException
 260          * @throws ParseException
 261          */
 262         private DateTime makeImportDateTime(ResultSet rs) throws SQLException,
 263                         ParseException {
 264                 DateTime importDateTime = null;
 265                 String importDateTimeStr = rs.getString("ImportDateTime");
 266                 if (isNotBlank(importDateTimeStr)){
 267                         importDateTimeStr = importDateTimeStr.substring(0,10);
 268                         DateTimeFormatter dayFormatter = DateTimeFormat.forPattern("dd.MM.yyyy");
 269
 270 //                                              DateTimeFormatter formatter = new DateTimeFormatterBuilder().
 271 //                                                              append;
 272                         DateTimeParser p = new DateTimeParser(dayFormatter);
 273                         importDateTime = p.parse(importDateTimeStr, Locale.GERMANY);
 274
 275                 }
 276                 return importDateTime;
 277         }
 278
 279
 280
 281         private Sequence makeSequence(ResultSet rs, DnaSample dnaSample, int dnaFactId, DateTime importDateTime) throws SQLException {
 282                 String sequenceStr = rs.getString("PlainSequence");
 283                 Integer seqLen = nullSafeInt(rs, "SeqLen");
 284
 285                 if (sequenceStr == null){
 286                         logger.warn("PlainSequence is null. Id: " + dnaFactId);
 287                 }else{
 288                         if (seqLen == null){
 289                                 logger.warn("SeqLen is null for dnaFact: "  + dnaFactId);
 290                         }else if (sequenceStr.length() != seqLen){
 291                                 logger.warn("SeqLen (" + seqLen+ ") and OriginalLen ("+sequenceStr.length()+") differ for dnaFact: "  + dnaFactId);
 292                         }
 293                 }
 294
 295                 Sequence sequence = Sequence.NewInstance(sequenceStr, seqLen);
 296                 sequence.setCreated(importDateTime);
 297                 dnaSample.addSequence(sequence);
 298                 return sequence;
 299         }
 300
 301
 302
 303         /**
 304          * @param sequence2
 305          * @param rs
 306          * @param accessionStr
 307          * @param notesStr
 308          * @param sequence
 309          * @param importDateTime
 310          * @return
 311          * @throws SQLException
 312          */
 313         private void makeGenBankAccession(ResultSet rs, Sequence sequence, DateTime importDateTime, Integer dnaFactId) throws SQLException {
 314                 String accessionStr = rs.getString("Accession");
 315                 String notesStr = rs.getString("Notes");
 316                 String versionStr = rs.getString("Version");
 317
 318                 URI genBankUri = null;
 319                 if (StringUtils.isNotBlank(notesStr)){
 320                         if (notesStr.startsWith("http")){
 321                                 genBankUri = URI.create(notesStr);
 322                         }else{
 323                                 logger.warn("Notes do not start with URI: " +  notesStr);
 324                         }
 325                 }
 326
 327                 if (isNotBlank(accessionStr) || genBankUri != null){
 328                         if (accessionStr != null && accessionStr.trim().equals("")){
 329                                 accessionStr = null;
 330                         }
 331                         if (isGenBankAccessionNumber(accessionStr, versionStr, genBankUri, dnaFactId) || genBankUri != null){
 332                                 sequence.setGeneticAccessionNumber(accessionStr);
 333                         }
 334                 }
 335         }
 336
 337         private boolean isGenBankAccessionNumber(String accessionStr, String versionStr, URI genBankUri, Integer dnaFactId) {
 338                 boolean isGenBankAccessionNumber = accessionStr.matches("[A-Z]{2}\\d{6}");
 339                 boolean versionHasGenBankPart = versionStr.matches(".*GI:.*");
 340                 if (isGenBankAccessionNumber && versionHasGenBankPart){
 341                         return true;
 342                 }else {
 343                         if (genBankUri != null){
 344                                 logger.warn("GenBank Uri exists but accession or version have been identified to use GenBank syntax. DNAFactID: " + dnaFactId);
 345                         }
 346                         if(isGenBankAccessionNumber || versionHasGenBankPart){
 347                                 logger.warn("Either accession ("+ accessionStr +") or version ("+versionStr+") use GenBank syntax but the other does not. DNAFactID: " + dnaFactId);
 348                         }
 349                         return false;
 350                 }
 351         }
 352
 353
 354
 355         protected String getDerivedUnitNameSpace(){
 356                 return ECO_FACT_DERIVED_UNIT_NAMESPACE;
 357         }
 358
 359         protected String getFieldObservationNameSpace(){
 360                 return ECO_FACT_FIELD_OBSERVATION_NAMESPACE;
 361         }
 362
 363         @Override
 364         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
 365                 String nameSpace;
 366                 Class<?> cdmClass;
 367                 Set<String> idSet;
 368                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 369
 370                 try{
 371                         Set<String> taxonIdSet = new HashSet<String>();
 372
 373                         Set<String> ecoFactFkSet = new HashSet<String>();
 374
 375                         while (rs.next()){
 376                                 handleForeignKey(rs, taxonIdSet, "taxonId");
 377                                 handleForeignKey(rs, ecoFactFkSet, "ecoFactId");
 378                         }
 379
 380                         //taxon map
 381                         nameSpace = BerlinModelTaxonImport.NAMESPACE;
 382                         cdmClass = TaxonBase.class;
 383                         idSet = taxonIdSet;
 384                         Map<String, TaxonBase> objectMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 385                         result.put(nameSpace, objectMap);
 386
 387
 388                         //eco fact derived unit map
 389                         nameSpace = AlgaTerraFactEcologyImport.ECO_FACT_DERIVED_UNIT_NAMESPACE;
 390                         cdmClass = DerivedUnit.class;
 391                         idSet = ecoFactFkSet;
 392                         Map<String, DerivedUnit> derivedUnitMap = (Map<String, DerivedUnit>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 393                         result.put(nameSpace, derivedUnitMap);
 394
 395                 } catch (SQLException e) {
 396                         throw new RuntimeException(e);
 397                 }
 398                 return result;
 399         }
 400
 401         @Override
 402         protected boolean doCheck(BerlinModelImportState state){
 403                 IOValidator<BerlinModelImportState> validator = new AlgaTerraDnaImportValidator();
 404                 return validator.validate(state);
 405         }
 406
 407         @Override
 408         protected boolean isIgnore(BerlinModelImportState state){
 409                 return ! ((AlgaTerraImportState)state).getAlgaTerraConfigurator().isDoDna();
 410         }
 411
 412 }