cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/out/PesiSourceExport.java

   1 // $Id$
   2 /**
   3 * Copyright (C) 2009 EDIT
   4 * European Distributed Institute of Taxonomy
   5 * http://www.e-taxonomy.eu
   6 *
   7 * The contents of this file are subject to the Mozilla Public License Version 1.1
   8 * See LICENSE.TXT at the top of this package for the full license terms.
   9 */
  10 package eu.etaxonomy.cdm.io.pesi.out;
  11
  12 import java.sql.SQLException;
  13 import java.util.ArrayList;
  14 import java.util.HashSet;
  15 import java.util.List;
  16 import java.util.Set;
  17
  18 import org.apache.log4j.Logger;
  19 import org.springframework.stereotype.Component;
  20 import org.springframework.transaction.TransactionStatus;
  21
  22 import eu.etaxonomy.cdm.io.common.Source;
  23 import eu.etaxonomy.cdm.io.common.IExportConfigurator.DO_REFERENCES;
  24 import eu.etaxonomy.cdm.io.common.mapping.out.DbExtensionMapper;
  25 import eu.etaxonomy.cdm.io.common.mapping.out.DbStringMapper;
  26 import eu.etaxonomy.cdm.io.common.mapping.out.DbTimePeriodMapper;
  27 import eu.etaxonomy.cdm.io.common.mapping.out.DbUriMapper;
  28 import eu.etaxonomy.cdm.io.common.mapping.out.IdMapper;
  29 import eu.etaxonomy.cdm.io.common.mapping.out.MethodMapper;
  30 import eu.etaxonomy.cdm.io.pesi.erms.ErmsTransformer;
  31 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  32 import eu.etaxonomy.cdm.model.common.CdmBase;
  33 import eu.etaxonomy.cdm.model.common.ExtensionType;
  34 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  35 import eu.etaxonomy.cdm.model.reference.Reference;
  36 import eu.etaxonomy.cdm.model.reference.ReferenceType;
  37
  38 /**
  39  * The export class for {@link eu.etaxonomy.cdm.model.reference.Reference References}.<p>
  40  * Inserts into DataWarehouse database table <code>Source</code>.
  41  * @author e.-m.lee
  42  * @date 11.02.2010
  43  *
  44  */
  45 @Component
  46 public class PesiSourceExport extends PesiExportBase {
  47         private static final Logger logger = Logger.getLogger(PesiSourceExport.class);
  48         private static final Class<? extends CdmBase> standardMethodParameter = Reference.class;
  49
  50         private static int modCount = 1000;
  51         public static final String dbTableName = "Source";
  52         private static final String pluralString = "Sources";
  53         List<Integer> storedSourceIds = new ArrayList<Integer>();
  54
  55         public PesiSourceExport() {
  56                 super();
  57         }
  58
  59         /* (non-Javadoc)
  60          * @see eu.etaxonomy.cdm.io.pesi.out.PesiExportBase#getStandardMethodParameter()
  61          */
  62         @Override
  63         public Class<? extends CdmBase> getStandardMethodParameter() {
  64                 return standardMethodParameter;
  65         }
  66
  67         /* (non-Javadoc)
  68          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
  69          */
  70         @Override
  71         protected boolean doCheck(PesiExportState state) {
  72                 boolean result = true;
  73                 return result;
  74         }
  75
  76         /**
  77          * Checks whether a sourceId was stored already.
  78          * @param sourceId
  79          * @return
  80          */
  81         protected boolean isStoredSourceId(Integer sourceId) {
  82                 if (storedSourceIds.contains(sourceId)) {
  83                         return true;
  84                 } else {
  85                         return false;
  86                 }
  87         }
  88
  89         /**
  90          * Adds a sourceId to the list of storedSourceIds.
  91          * @param sourceId
  92          */
  93         protected void addToStoredSourceIds(Integer sourceId) {
  94                 if (! storedSourceIds.contains(sourceId)) {
  95                         this.storedSourceIds.add(sourceId);
  96                 }
  97         }
  98
  99         /* (non-Javadoc)
 100          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IoStateBase)
 101          */
 102         @Override
 103         protected void doInvoke(PesiExportState state) {
 104                 try{
 105                         logger.info("*** Started Making " + pluralString + " ...");
 106
 107                         PesiExportConfigurator pesiExportConfigurator = state.getConfig();
 108
 109                         // Get the limit for objects to save within a single transaction.
 110                         int limit = pesiExportConfigurator.getLimitSave();
 111
 112                         // Stores whether this invoke was successful or not.
 113                         boolean success = true ;
 114
 115                         // PESI: Clear the database table Source.
 116                         //doDelete(state);  -> done by stored procedure
 117
 118                         // Get specific mappings: (CDM) Reference -> (PESI) Source
 119                         PesiExportMapping mapping = getMapping();
 120
 121                         // Initialize the db mapper
 122                         mapping.initialize(state);
 123
 124                         // Create the Sources
 125                         int count = 0;
 126                         int pastCount = 0;
 127                         TransactionStatus txStatus = null;
 128                         List<Reference> list = null;
 129
 130 //                      logger.error("PHASE 1...");
 131                         // Start transaction
 132                         txStatus = startTransaction(true);
 133                         logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") ...");
 134                         while ((list = getReferenceService().list(null, limit, count, null, null)).size() > 0) {
 135
 136                                 logger.debug("Fetched " + list.size() + " " + pluralString + ". Exporting...");
 137                                 for (Reference<?> reference : list) {
 138                                         doCount(count++, modCount, pluralString);
 139                                         success &= mapping.invoke(reference);
 140                                 }
 141
 142                                 // Commit transaction
 143                                 commitTransaction(txStatus);
 144                                 logger.debug("Committed transaction.");
 145                                 logger.info("Exported " + (count - pastCount) + " " + pluralString + ". Total: " + count);
 146                                 pastCount = count;
 147
 148                                 // Start transaction
 149                                 txStatus = startTransaction(true);
 150                                 logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") ...");
 151                         }
 152                         if (list.size() == 0) {
 153                                 logger.info("No " + pluralString + " left to fetch.");
 154                         }
 155                         // Commit transaction
 156                         commitTransaction(txStatus);
 157                         logger.info("Committed transaction.");
 158
 159                         logger.info("*** Finished Making " + pluralString + " ..." + getSuccessString(success));
 160
 161                         if (!success){
 162                                 state.setUnsuccessfull();
 163                         }
 164                         return;
 165                 } catch (SQLException e) {
 166                         e.printStackTrace();
 167                         logger.error(e.getMessage());
 168                         state.setUnsuccessfull();
 169                         return;
 170                 }
 171         }
 172
 173         /**
 174          * Deletes all entries of database tables related to <code>Source</code>.
 175          * @param state The {@link PesiExportState PesiExportState}.
 176          * @return Whether the delete operation was successful or not.
 177          */
 178         protected boolean doDelete(PesiExportState state) {
 179                 PesiExportConfigurator pesiConfig = (PesiExportConfigurator) state.getConfig();
 180
 181                 String sql;
 182                 Source destination =  pesiConfig.getDestination();
 183
 184                 // Clear Occurrences
 185                 sql = "DELETE FROM Occurrence";
 186                 destination.setQuery(sql);
 187                 destination.update(sql);
 188
 189                 // Clear Taxa
 190                 sql = "DELETE FROM Taxon";
 191                 destination.setQuery(sql);
 192                 destination.update(sql);
 193
 194                 // Clear Sources
 195                 sql = "DELETE FROM " + dbTableName;
 196                 destination.setQuery(sql);
 197                 destination.update(sql);
 198
 199                 return true;
 200         }
 201
 202         /**
 203          * Returns the <code>IMIS_Id</code> attribute.
 204          * @param reference The {@link Reference Reference}.
 205          * @return The <code>IMIS_Id</code> attribute.
 206          * @see MethodMapper
 207          */
 208         @SuppressWarnings("unused")
 209         private static Integer getIMIS_Id(Reference<?> reference) {
 210                 return null;
 211         }
 212
 213         /**
 214          * Returns the <code>SourceCategoryFK</code> attribute.
 215          * @param reference The {@link Reference Reference}.
 216          * @return The <code>SourceCategoryFK</code> attribute.
 217          * @see MethodMapper
 218          */
 219         @SuppressWarnings("unused")
 220         private static Integer getSourceCategoryFK(Reference<?> reference) {
 221                 Integer result = null;
 222                 try {
 223                 result = PesiTransformer.reference2SourceCategoryFK(reference);
 224                 } catch (Exception e) {
 225                         e.printStackTrace();
 226                 }
 227                 return result;
 228         }
 229
 230         /**
 231          * Returns the <code>SourceCategoryCache</code> attribute.
 232          * @param reference The {@link Reference Reference}.
 233          * @return The <code>SourceCategoryCache</code> attribute.
 234          * @see MethodMapper
 235          */
 236         @SuppressWarnings("unused")
 237         private static String getSourceCategoryCache(Reference<?> reference, PesiExportState state) {
 238                 return state.getTransformer().getCacheByReference(reference);
 239         }
 240
 241         /**
 242          * Returns the <code>Name</code> attribute. The corresponding CDM attribute is <code>title</code>.
 243          * @param reference The {@link Reference Reference}.
 244          * @return The <code>Name</code> attribute.
 245          * @see MethodMapper
 246          */
 247         @SuppressWarnings("unused")
 248         private static String getName(Reference<?> reference) {
 249                 if (reference != null) {
 250                         return reference.getTitleCache(); // was getTitle()
 251                 } else {
 252                         return null;
 253                 }
 254         }
 255
 256         /**
 257          * Returns the <code>AuthorString</code> attribute. The corresponding CDM attribute is the <code>titleCache</code> of an <code>authorTeam</code>.
 258          * @param reference The {@link Reference Reference}.
 259          * @return The <code>AuthorString</code> attribute.
 260          * @see MethodMapper
 261          */
 262         @SuppressWarnings("unused")
 263         private static String getAuthorString(Reference<?> reference) {
 264                 String result = null;
 265
 266                 try {
 267                 if (reference != null) {
 268                         TeamOrPersonBase team = reference.getAuthorTeam();
 269                         if (team != null) {
 270                                 result = team.getTitleCache();
 271 //                              result = team.getNomenclaturalTitle();
 272                         } else {
 273                                 result = null;
 274                         }
 275                 }
 276                 } catch (Exception e) {
 277                         e.printStackTrace();
 278                 }
 279
 280                 return result;
 281         }
 282
 283         /**
 284          * Returns the <code>NomRefCache</code> attribute. The corresponding CDM attribute is <code>titleCache</code>.
 285          * @param reference The {@link Reference Reference}.
 286          * @return The <code>NomRefCache</code> attribute.
 287          * @see MethodMapper
 288          */
 289         @SuppressWarnings("unused")
 290         private static String getNomRefCache(Reference<?> reference) {
 291                 return null;
 292 //              if (reference != null) {
 293 //                      return reference.getTitleCache();
 294 //              } else {
 295 //                      return null;
 296 //              }
 297         }
 298
 299         /**
 300          * Returns the <code>Notes</code> attribute.
 301          * @param reference The {@link Reference Reference}.
 302          * @return The <code>Notes</code> attribute.
 303          * @see MethodMapper
 304          */
 305         @SuppressWarnings("unused")
 306         private static String getNotes(Reference<?> reference) {
 307                 // TODO
 308                 return null;
 309         }
 310
 311         /**
 312          * Returns the <code>RefIdInSource</code> attribute.
 313          * @param reference The {@link Reference Reference}.
 314          * @return The <code>RefIdInSource</code> attribute.
 315          * @see MethodMapper
 316          */
 317         @SuppressWarnings("unused")
 318         private static String getRefIdInSource(Reference<?> reference) {
 319                 String result = null;
 320
 321                 try {
 322                 if (reference != null) {
 323                         Set<IdentifiableSource> sourceAll = reference.getSources();
 324                         Set<IdentifiableSource> sourceCandidates = filterOriginalPesiDbSources(sourceAll);
 325
 326                         if (sourceCandidates.size() == 1) {
 327                                 result = sourceCandidates.iterator().next().getIdInSource();
 328                         } else if (sourceCandidates.size() > 1) {
 329                                 logger.warn("Reference for RefIdInSource has multiple IdentifiableSources which are candidates for a PESI originalDbSource. RefIdInSource can't be determined correctly and will be left out: " + reference.getUuid() + " (" + reference.getTitleCache() + ")");
 330                                 int count = 1;
 331 //                              for (IdentifiableSource source : sources) {
 332 //                                      result += source.getIdInSource();
 333 //                                      if (count < sources.size()) {
 334 //                                              result += "; ";
 335 //                                      }
 336 //                                      count++;
 337 //                              }
 338                         }
 339                 }
 340                 } catch (Exception e) {
 341                         e.printStackTrace();
 342                 }
 343
 344                 return result;
 345         }
 346
 347         private static Set<IdentifiableSource> filterOriginalPesiDbSources(
 348                         Set<IdentifiableSource> sourceAll) {
 349                 Set<IdentifiableSource> sourceCandidates = new HashSet<IdentifiableSource>();
 350                 for (IdentifiableSource source : sourceAll){
 351                         if (isOriginalPesiDbSource(source)){
 352                                 sourceCandidates.add(source);
 353                         }
 354                 }
 355                 return sourceCandidates;
 356         }
 357
 358         private static boolean isOriginalPesiDbSource(IdentifiableSource source) {
 359                 return (source.getCitation() != null) &&
 360                                 source.getCitation().getType().equals(ReferenceType.Database);
 361         }
 362
 363         /**
 364          * Returns the <code>OriginalDB</code> attribute. The corresponding CDM attribute is the <code>titleCache</code> of a <code>citation</code>.
 365          * @param reference The {@link Reference Reference}.
 366          * @return The <code>OriginalDB</code> attribute.
 367          * @see MethodMapper
 368          */
 369         @SuppressWarnings("unused")
 370         private static String getOriginalDB(Reference<?> reference) {
 371                 String result = "";
 372
 373                 try {
 374                 if (reference != null) {
 375                         Set<IdentifiableSource> sourcesAll = reference.getSources();
 376                         Set<IdentifiableSource> sourceCandidates = filterOriginalPesiDbSources(sourcesAll);
 377
 378                         if (sourceCandidates.size() == 1) {
 379                                 Reference citation = sourceCandidates.iterator().next().getCitation();
 380                                 if (citation != null) {
 381                                         result = PesiTransformer.databaseString2Abbreviation(citation.getTitleCache()); //or just title
 382                                 } else {
 383                                         logger.warn("OriginalDB can not be determined because the citation of this source is NULL: " + sourceCandidates.iterator().next().getUuid());
 384                                 }
 385                         } else if (sourceCandidates.size() > 1) {
 386                                 logger.warn("Taxon has multiple IdentifiableSources: " + reference.getUuid() + " (" + reference.getTitleCache() + ")");
 387                                 int count = 1;
 388                                 for (IdentifiableSource source : sourceCandidates) {
 389                                         Reference citation = source.getCitation();
 390                                         if (citation != null) {
 391                                                 result += PesiTransformer.databaseString2Abbreviation(citation.getTitleCache());
 392                                                 if (count < sourceCandidates.size()) {
 393                                                         result += "; ";
 394                                                 }
 395                                                 count++;
 396                                         }
 397                                 }
 398                         } else {
 399                                 result = null;
 400                         }
 401                 }
 402                 } catch (Exception e) {
 403                         e.printStackTrace();
 404                 }
 405
 406                 return result;
 407         }
 408
 409         /* (non-Javadoc)
 410          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IoStateBase)
 411          */
 412         @Override
 413         protected boolean isIgnore(PesiExportState state) {
 414                 return ! state.getConfig().getDoReferences().equals(DO_REFERENCES.ALL);
 415         }
 416
 417         /**
 418          * Returns the CDM to PESI specific export mappings.
 419          * @return The {@link PesiExportMapping PesiExportMapping}.
 420          */
 421         private PesiExportMapping getMapping() {
 422                 PesiExportMapping mapping = new PesiExportMapping(dbTableName);
 423                 ExtensionType extensionType = null;
 424
 425                 mapping.addMapper(IdMapper.NewInstance("SourceId"));
 426
 427                 // IMIS_Id
 428                 extensionType = (ExtensionType)getTermService().find(ErmsTransformer.IMIS_UUID);
 429                 if (extensionType != null) {
 430                         mapping.addMapper(DbExtensionMapper.NewInstance(extensionType, "IMIS_Id"));
 431                 } else {
 432                         mapping.addMapper(MethodMapper.NewInstance("IMIS_Id", this));
 433                 }
 434
 435                 mapping.addMapper(MethodMapper.NewInstance("SourceCategoryFK", this));
 436                 mapping.addMapper(MethodMapper.NewInstance("SourceCategoryCache", this, Reference.class, PesiExportState.class));
 437                 mapping.addMapper(MethodMapper.NewInstance("Name", this));
 438                 mapping.addMapper(DbStringMapper.NewInstance("referenceAbstract", "Abstract"));
 439                 mapping.addMapper(DbStringMapper.NewInstance("title", "Title"));
 440                 mapping.addMapper(MethodMapper.NewInstance("AuthorString", this));
 441                 mapping.addMapper(DbTimePeriodMapper.NewInstance("datePublished", "RefYear"));
 442                 mapping.addMapper(MethodMapper.NewInstance("NomRefCache", this));
 443                 mapping.addMapper(DbUriMapper.NewInstance("uri", "Link"));
 444                 mapping.addMapper(MethodMapper.NewInstance("Notes", this));
 445                 mapping.addMapper(MethodMapper.NewInstance("RefIdInSource", this));
 446                 mapping.addMapper(MethodMapper.NewInstance("OriginalDB", this));
 447
 448                 return mapping;
 449         }
 450
 451 }