cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/out/PesiSourceExport.java

   1 // $Id$
   2 /**
   3 * Copyright (C) 2009 EDIT
   4 * European Distributed Institute of Taxonomy
   5 * http://www.e-taxonomy.eu
   6 *
   7 * The contents of this file are subject to the Mozilla Public License Version 1.1
   8 * See LICENSE.TXT at the top of this package for the full license terms.
   9 */
  10 package eu.etaxonomy.cdm.io.pesi.out;
  11
  12 import java.sql.SQLException;
  13 import java.util.ArrayList;
  14 import java.util.HashSet;
  15 import java.util.List;
  16 import java.util.Set;
  17
  18 import org.apache.log4j.Logger;
  19 import org.springframework.stereotype.Component;
  20 import org.springframework.transaction.TransactionStatus;
  21
  22 import eu.etaxonomy.cdm.io.common.Source;
  23 import eu.etaxonomy.cdm.io.common.IExportConfigurator.DO_REFERENCES;
  24 import eu.etaxonomy.cdm.io.common.mapping.out.DbExtensionMapper;
  25 import eu.etaxonomy.cdm.io.common.mapping.out.DbStringMapper;
  26 import eu.etaxonomy.cdm.io.common.mapping.out.DbTimePeriodMapper;
  27 import eu.etaxonomy.cdm.io.common.mapping.out.DbUriMapper;
  28 import eu.etaxonomy.cdm.io.common.mapping.out.IdMapper;
  29 import eu.etaxonomy.cdm.io.common.mapping.out.MethodMapper;
  30 import eu.etaxonomy.cdm.io.pesi.erms.ErmsTransformer;
  31 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  32 import eu.etaxonomy.cdm.model.common.CdmBase;
  33 import eu.etaxonomy.cdm.model.common.ExtensionType;
  34 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  35 import eu.etaxonomy.cdm.model.reference.Reference;
  36 import eu.etaxonomy.cdm.model.reference.ReferenceType;
  37
  38 /**
  39  * The export class for {@link eu.etaxonomy.cdm.model.reference.Reference References}.<p>
  40  * Inserts into DataWarehouse database table <code>Source</code>.
  41  * @author e.-m.lee
  42  * @date 11.02.2010
  43  *
  44  */
  45 @Component
  46 public class PesiSourceExport extends PesiExportBase {
  47         private static final Logger logger = Logger.getLogger(PesiSourceExport.class);
  48         private static final Class<? extends CdmBase> standardMethodParameter = Reference.class;
  49
  50         private static int modCount = 1000;
  51         private static final String dbTableName = "Source";
  52         private static final String pluralString = "Sources";
  53         List<Integer> storedSourceIds = new ArrayList<Integer>();
  54
  55         public PesiSourceExport() {
  56                 super();
  57         }
  58
  59         /* (non-Javadoc)
  60          * @see eu.etaxonomy.cdm.io.pesi.out.PesiExportBase#getStandardMethodParameter()
  61          */
  62         @Override
  63         public Class<? extends CdmBase> getStandardMethodParameter() {
  64                 return standardMethodParameter;
  65         }
  66
  67         /* (non-Javadoc)
  68          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
  69          */
  70         @Override
  71         protected boolean doCheck(PesiExportState state) {
  72                 boolean result = true;
  73                 return result;
  74         }
  75
  76         /**
  77          * Checks whether a sourceId was stored already.
  78          * @param sourceId
  79          * @return
  80          */
  81         protected boolean isStoredSourceId(Integer sourceId) {
  82                 if (storedSourceIds.contains(sourceId)) {
  83                         return true;
  84                 } else {
  85                         return false;
  86                 }
  87         }
  88
  89         /**
  90          * Adds a sourceId to the list of storedSourceIds.
  91          * @param sourceId
  92          */
  93         protected void addToStoredSourceIds(Integer sourceId) {
  94                 if (! storedSourceIds.contains(sourceId)) {
  95                         this.storedSourceIds.add(sourceId);
  96                 }
  97         }
  98
  99         /* (non-Javadoc)
 100          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IoStateBase)
 101          */
 102         @Override
 103         protected void doInvoke(PesiExportState state) {
 104                 try{
 105                         logger.info("*** Started Making " + pluralString + " ...");
 106
 107                         PesiExportConfigurator pesiExportConfigurator = state.getConfig();
 108
 109                         // Get the limit for objects to save within a single transaction.
 110                         int limit = pesiExportConfigurator.getLimitSave();
 111
 112                         // Stores whether this invoke was successful or not.
 113                         boolean success = true ;
 114
 115                         // PESI: Clear the database table Source.
 116                         doDelete(state);
 117
 118                         // Get specific mappings: (CDM) Reference -> (PESI) Source
 119                         PesiExportMapping mapping = getMapping();
 120
 121                         // Initialize the db mapper
 122                         mapping.initialize(state);
 123
 124                         // Create the Sources
 125                         int count = 0;
 126                         int pastCount = 0;
 127                         TransactionStatus txStatus = null;
 128                         List<Reference> list = null;
 129
 130 //                      logger.error("PHASE 1...");
 131                         // Start transaction
 132                         txStatus = startTransaction(true);
 133                         logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") ...");
 134                         while ((list = getReferenceService().list(null, limit, count, null, null)).size() > 0) {
 135
 136                                 logger.debug("Fetched " + list.size() + " " + pluralString + ". Exporting...");
 137                                 for (Reference<?> reference : list) {
 138                                         doCount(count++, modCount, pluralString);
 139                                         success &= mapping.invoke(reference);
 140                                 }
 141
 142                                 // Commit transaction
 143                                 commitTransaction(txStatus);
 144                                 logger.debug("Committed transaction.");
 145                                 logger.info("Exported " + (count - pastCount) + " " + pluralString + ". Total: " + count);
 146                                 pastCount = count;
 147
 148                                 // Start transaction
 149                                 txStatus = startTransaction(true);
 150                                 logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") ...");
 151                         }
 152                         if (list.size() == 0) {
 153                                 logger.info("No " + pluralString + " left to fetch.");
 154                         }
 155                         // Commit transaction
 156                         commitTransaction(txStatus);
 157                         logger.info("Committed transaction.");
 158
 159                         logger.info("*** Finished Making " + pluralString + " ..." + getSuccessString(success));
 160
 161                         if (!success){
 162                                 state.setUnsuccessfull();
 163                         }
 164                         return;
 165                 } catch (SQLException e) {
 166                         e.printStackTrace();
 167                         logger.error(e.getMessage());
 168                         state.setUnsuccessfull();
 169                         return;
 170                 }
 171         }
 172
 173         /**
 174          * Deletes all entries of database tables related to <code>Source</code>.
 175          * @param state The {@link PesiExportState PesiExportState}.
 176          * @return Whether the delete operation was successful or not.
 177          */
 178         protected boolean doDelete(PesiExportState state) {
 179                 PesiExportConfigurator pesiConfig = (PesiExportConfigurator) state.getConfig();
 180
 181                 String sql;
 182                 Source destination =  pesiConfig.getDestination();
 183
 184                 // Clear Occurrences
 185                 sql = "DELETE FROM Occurrence";
 186                 destination.setQuery(sql);
 187                 destination.update(sql);
 188
 189                 // Clear Taxa
 190                 sql = "DELETE FROM Taxon";
 191                 destination.setQuery(sql);
 192                 destination.update(sql);
 193
 194                 // Clear Sources
 195                 sql = "DELETE FROM " + dbTableName;
 196                 destination.setQuery(sql);
 197                 destination.update(sql);
 198
 199                 return true;
 200         }
 201
 202         /**
 203          * Returns the <code>IMIS_Id</code> attribute.
 204          * @param reference The {@link Reference Reference}.
 205          * @return The <code>IMIS_Id</code> attribute.
 206          * @see MethodMapper
 207          */
 208         @SuppressWarnings("unused")
 209         private static Integer getIMIS_Id(Reference<?> reference) {
 210                 return null;
 211         }
 212
 213         /**
 214          * Returns the <code>SourceCategoryFK</code> attribute.
 215          * @param reference The {@link Reference Reference}.
 216          * @return The <code>SourceCategoryFK</code> attribute.
 217          * @see MethodMapper
 218          */
 219         @SuppressWarnings("unused")
 220         private static Integer getSourceCategoryFK(Reference<?> reference) {
 221                 Integer result = null;
 222                 try {
 223                 result = PesiTransformer.reference2SourceCategoryFK(reference);
 224                 } catch (Exception e) {
 225                         e.printStackTrace();
 226                 }
 227                 return result;
 228         }
 229
 230         /**
 231          * Returns the <code>SourceCategoryCache</code> attribute.
 232          * @param reference The {@link Reference Reference}.
 233          * @return The <code>SourceCategoryCache</code> attribute.
 234          * @see MethodMapper
 235          */
 236         @SuppressWarnings("unused")
 237         private static String getSourceCategoryCache(Reference<?> reference) {
 238                 String result = null;
 239                 try {
 240                 result = PesiTransformer.getSourceCategoryCache(reference);
 241                 } catch (Exception e) {
 242                         e.printStackTrace();
 243                 }
 244                 return result;
 245         }
 246
 247         /**
 248          * Returns the <code>Name</code> attribute. The corresponding CDM attribute is <code>title</code>.
 249          * @param reference The {@link Reference Reference}.
 250          * @return The <code>Name</code> attribute.
 251          * @see MethodMapper
 252          */
 253         @SuppressWarnings("unused")
 254         private static String getName(Reference<?> reference) {
 255                 if (reference != null) {
 256                         return reference.getTitleCache(); // was getTitle()
 257                 } else {
 258                         return null;
 259                 }
 260         }
 261
 262         /**
 263          * Returns the <code>AuthorString</code> attribute. The corresponding CDM attribute is the <code>titleCache</code> of an <code>authorTeam</code>.
 264          * @param reference The {@link Reference Reference}.
 265          * @return The <code>AuthorString</code> attribute.
 266          * @see MethodMapper
 267          */
 268         @SuppressWarnings("unused")
 269         private static String getAuthorString(Reference<?> reference) {
 270                 String result = null;
 271
 272                 try {
 273                 if (reference != null) {
 274                         TeamOrPersonBase team = reference.getAuthorTeam();
 275                         if (team != null) {
 276                                 result = team.getTitleCache();
 277 //                              result = team.getNomenclaturalTitle();
 278                         } else {
 279                                 result = null;
 280                         }
 281                 }
 282                 } catch (Exception e) {
 283                         e.printStackTrace();
 284                 }
 285
 286                 return result;
 287         }
 288
 289         /**
 290          * Returns the <code>NomRefCache</code> attribute. The corresponding CDM attribute is <code>titleCache</code>.
 291          * @param reference The {@link Reference Reference}.
 292          * @return The <code>NomRefCache</code> attribute.
 293          * @see MethodMapper
 294          */
 295         @SuppressWarnings("unused")
 296         private static String getNomRefCache(Reference<?> reference) {
 297                 return null;
 298 //              if (reference != null) {
 299 //                      return reference.getTitleCache();
 300 //              } else {
 301 //                      return null;
 302 //              }
 303         }
 304
 305         /**
 306          * Returns the <code>Notes</code> attribute.
 307          * @param reference The {@link Reference Reference}.
 308          * @return The <code>Notes</code> attribute.
 309          * @see MethodMapper
 310          */
 311         @SuppressWarnings("unused")
 312         private static String getNotes(Reference<?> reference) {
 313                 // TODO
 314                 return null;
 315         }
 316
 317         /**
 318          * Returns the <code>RefIdInSource</code> attribute.
 319          * @param reference The {@link Reference Reference}.
 320          * @return The <code>RefIdInSource</code> attribute.
 321          * @see MethodMapper
 322          */
 323         @SuppressWarnings("unused")
 324         private static String getRefIdInSource(Reference<?> reference) {
 325                 String result = null;
 326
 327                 try {
 328                 if (reference != null) {
 329                         Set<IdentifiableSource> sourceAll = reference.getSources();
 330                         Set<IdentifiableSource> sourceCandidates = filterOriginalPesiDbSources(sourceAll);
 331
 332                         if (sourceCandidates.size() == 1) {
 333                                 result = sourceCandidates.iterator().next().getIdInSource();
 334                         } else if (sourceCandidates.size() > 1) {
 335                                 logger.warn("Reference for RefIdInSource has multiple IdentifiableSources which are candidates for a PESI originalDbSource. RefIdInSource can't be determined correctly and will be left out: " + reference.getUuid() + " (" + reference.getTitleCache() + ")");
 336                                 int count = 1;
 337 //                              for (IdentifiableSource source : sources) {
 338 //                                      result += source.getIdInSource();
 339 //                                      if (count < sources.size()) {
 340 //                                              result += "; ";
 341 //                                      }
 342 //                                      count++;
 343 //                              }
 344                         }
 345                 }
 346                 } catch (Exception e) {
 347                         e.printStackTrace();
 348                 }
 349
 350                 return result;
 351         }
 352
 353         private static Set<IdentifiableSource> filterOriginalPesiDbSources(
 354                         Set<IdentifiableSource> sourceAll) {
 355                 Set<IdentifiableSource> sourceCandidates = new HashSet<IdentifiableSource>();
 356                 for (IdentifiableSource source : sourceAll){
 357                         if (isOriginalPesiDbSource(source)){
 358                                 sourceCandidates.add(source);
 359                         }
 360                 }
 361                 return sourceCandidates;
 362         }
 363
 364         private static boolean isOriginalPesiDbSource(IdentifiableSource source) {
 365                 return (source.getCitation() != null) &&
 366                                 source.getCitation().getType().equals(ReferenceType.Database);
 367         }
 368
 369         /**
 370          * Returns the <code>OriginalDB</code> attribute. The corresponding CDM attribute is the <code>titleCache</code> of a <code>citation</code>.
 371          * @param reference The {@link Reference Reference}.
 372          * @return The <code>OriginalDB</code> attribute.
 373          * @see MethodMapper
 374          */
 375         @SuppressWarnings("unused")
 376         private static String getOriginalDB(Reference<?> reference) {
 377                 String result = "";
 378
 379                 try {
 380                 if (reference != null) {
 381                         Set<IdentifiableSource> sourcesAll = reference.getSources();
 382                         Set<IdentifiableSource> sourceCandidates = filterOriginalPesiDbSources(sourcesAll);
 383
 384                         if (sourceCandidates.size() == 1) {
 385                                 Reference citation = sourceCandidates.iterator().next().getCitation();
 386                                 if (citation != null) {
 387                                         result = PesiTransformer.databaseString2Abbreviation(citation.getTitleCache()); //or just title
 388                                 } else {
 389                                         logger.warn("OriginalDB can not be determined because the citation of this source is NULL: " + sourceCandidates.iterator().next().getUuid());
 390                                 }
 391                         } else if (sourceCandidates.size() > 1) {
 392                                 logger.warn("Taxon has multiple IdentifiableSources: " + reference.getUuid() + " (" + reference.getTitleCache() + ")");
 393                                 int count = 1;
 394                                 for (IdentifiableSource source : sourceCandidates) {
 395                                         Reference citation = source.getCitation();
 396                                         if (citation != null) {
 397                                                 result += PesiTransformer.databaseString2Abbreviation(citation.getTitleCache());
 398                                                 if (count < sourceCandidates.size()) {
 399                                                         result += "; ";
 400                                                 }
 401                                                 count++;
 402                                         }
 403                                 }
 404                         } else {
 405                                 result = null;
 406                         }
 407                 }
 408                 } catch (Exception e) {
 409                         e.printStackTrace();
 410                 }
 411
 412                 return result;
 413         }
 414
 415         /* (non-Javadoc)
 416          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IoStateBase)
 417          */
 418         @Override
 419         protected boolean isIgnore(PesiExportState state) {
 420                 return ! state.getConfig().getDoReferences().equals(DO_REFERENCES.ALL);
 421         }
 422
 423         /**
 424          * Returns the CDM to PESI specific export mappings.
 425          * @return The {@link PesiExportMapping PesiExportMapping}.
 426          */
 427         private PesiExportMapping getMapping() {
 428                 PesiExportMapping mapping = new PesiExportMapping(dbTableName);
 429                 ExtensionType extensionType = null;
 430
 431                 mapping.addMapper(IdMapper.NewInstance("SourceId"));
 432
 433                 // IMIS_Id
 434                 extensionType = (ExtensionType)getTermService().find(ErmsTransformer.IMIS_UUID);
 435                 if (extensionType != null) {
 436                         mapping.addMapper(DbExtensionMapper.NewInstance(extensionType, "IMIS_Id"));
 437                 } else {
 438                         mapping.addMapper(MethodMapper.NewInstance("IMIS_Id", this));
 439                 }
 440
 441                 mapping.addMapper(MethodMapper.NewInstance("SourceCategoryFK", this));
 442                 mapping.addMapper(MethodMapper.NewInstance("SourceCategoryCache", this));
 443                 mapping.addMapper(MethodMapper.NewInstance("Name", this));
 444                 mapping.addMapper(DbStringMapper.NewInstance("referenceAbstract", "Abstract"));
 445                 mapping.addMapper(DbStringMapper.NewInstance("title", "Title"));
 446                 mapping.addMapper(MethodMapper.NewInstance("AuthorString", this));
 447                 mapping.addMapper(DbTimePeriodMapper.NewInstance("datePublished", "RefYear"));
 448                 mapping.addMapper(MethodMapper.NewInstance("NomRefCache", this));
 449                 mapping.addMapper(DbUriMapper.NewInstance("uri", "Link"));
 450                 mapping.addMapper(MethodMapper.NewInstance("Notes", this));
 451                 mapping.addMapper(MethodMapper.NewInstance("RefIdInSource", this));
 452                 mapping.addMapper(MethodMapper.NewInstance("OriginalDB", this));
 453
 454                 return mapping;
 455         }
 456
 457 }