cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/DbImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.common;
  11
  12 import java.lang.reflect.Method;
  13 import java.sql.ResultSet;
  14 import java.sql.ResultSetMetaData;
  15 import java.sql.SQLException;
  16 import java.util.HashMap;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.commons.lang.StringUtils;
  22 import org.apache.log4j.Logger;
  23
  24 import eu.etaxonomy.cdm.common.CdmUtils;
  25 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  26 import eu.etaxonomy.cdm.model.common.Annotation;
  27 import eu.etaxonomy.cdm.model.common.ISourceable;
  28 import eu.etaxonomy.cdm.model.common.Language;
  29 import eu.etaxonomy.cdm.model.common.User;
  30
  31 /**
  32  * @author a.mueller
  33  * @created 20.03.2008
  34  * @version 1.0
  35  */
  36 public abstract class DbImportBase<STATE extends DbImportStateBase<CONFIG, STATE>, CONFIG extends DbImportConfiguratorBase<STATE>> extends CdmImportBase<CONFIG, STATE> implements ICdmIO<STATE>, IPartitionedIO<STATE> {
  37         private static final Logger logger = Logger.getLogger(DbImportBase.class);
  38
  39         private String dbTableName ;
  40         private String pluralString;
  41
  42         public DbImportBase(String tableName, String pluralString) {
  43                 super();
  44                 this.dbTableName = tableName;
  45                 this.pluralString = pluralString;
  46         }
  47
  48         protected void doInvoke(STATE state){
  49                         //      String strTeamStore = ICdmIO.TEAM_STORE;
  50                         CONFIG config = state.getConfig();
  51                         Source source = config.getSource();
  52                         boolean success = true ;
  53
  54                         logger.info("start make " + getPluralString() + " ...");
  55
  56                         String strIdQuery = getIdQuery(state);
  57                         String strRecordQuery = getRecordQuery(config);
  58
  59                         int recordsPerTransaction = config.getRecordsPerTransaction();
  60                         try{
  61                                 ResultSetPartitioner<STATE> partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery, strRecordQuery, recordsPerTransaction);
  62                                 while (partitioner.nextPartition()){
  63                                         try {
  64                                                 partitioner.doPartition(this, state);
  65                                         } catch (Exception e) {
  66                                                 e.printStackTrace();
  67                                                 success = false;
  68                                         }
  69                                 }
  70                         } catch (SQLException e) {
  71                                 logger.error("SQLException:" +  e);
  72                                 state.setUnsuccessfull();
  73                         }
  74
  75                         logger.info("end make " + getPluralString() + " ... " + getSuccessString(success));
  76                         if (success == false){
  77                                 state.setUnsuccessfull();
  78                         }
  79                         return;
  80         }
  81
  82
  83         /**
  84          * @return
  85          */
  86         protected abstract String getRecordQuery(CONFIG config);
  87
  88         /**
  89          * @return
  90          */
  91         protected abstract String getIdQuery(STATE state);
  92
  93
  94         /**
  95          * @return
  96          */
  97         protected String getTableName() {
  98                 return dbTableName;
  99         }
 100
 101
 102         /* (non-Javadoc)
 103          * @see eu.etaxonomy.cdm.io.common.IPartitionedIO#getPluralString()
 104          */
 105         public String getPluralString() {
 106                 return pluralString;
 107         }
 108
 109
 110         /**
 111          * @param state
 112          * @param sourceable
 113          * @param id
 114          * @param namespace
 115          * @return
 116          */
 117         protected boolean doId(STATE state, ISourceable sourceable, long id, String namespace) {
 118                 return ImportHelper.setOriginalSource(sourceable, state.getTransactionalSourceReference() , id, namespace);
 119         }
 120
 121         /**
 122          * @param state
 123          * @param sourceable
 124          * @param id
 125          * @param namespace
 126          * @return
 127          */
 128         protected boolean doId(STATE state, ISourceable sourceable, String id, String namespace) {
 129                 return ImportHelper.setOriginalSource(sourceable, state.getTransactionalSourceReference() , id, namespace);
 130         }
 131
 132
 133
 134         /**
 135          * Adds a note to the annotatable entity.
 136          * Nothing happens if annotatableEntity is <code>null</code> or notes is empty or <code>null</code>.
 137          * @param annotatableEntity
 138          * @param notes
 139          */
 140         protected void doNotes(AnnotatableEntity annotatableEntity, String notes) {
 141                 if (StringUtils.isNotBlank(notes) && annotatableEntity != null ){
 142                         String notesString = String.valueOf(notes);
 143                         if (notesString.length() > 65530 ){
 144                                 notesString = notesString.substring(0, 65530) + "...";
 145                                 logger.warn("Notes string is longer than 65530 and was truncated: " + annotatableEntity);
 146                         }
 147                         Annotation notesAnnotation = Annotation.NewInstance(notesString, Language.DEFAULT());
 148                         //notesAnnotation.setAnnotationType(AnnotationType.EDITORIAL());
 149                         //notes.setCommentator(bmiConfig.getCommentator());
 150                         annotatableEntity.addAnnotation(notesAnnotation);
 151                 }
 152         }
 153
 154
 155         protected User getUser(STATE state, String userString){
 156                 if (CdmUtils.isBlank(userString)){
 157                         return null;
 158                 }
 159                 userString = userString.trim();
 160
 161                 User user = state.getUser(userString);
 162                 if (user == null){
 163                         user = getTransformedUser(userString,state);
 164                 }
 165                 if (user == null){
 166                         user = makeNewUser(userString, state);
 167                 }
 168                 if (user == null){
 169                         logger.warn("User is null");
 170                 }
 171                 return user;
 172         }
 173
 174         private User getTransformedUser(String userString, STATE state){
 175                 Method method = state.getConfig().getUserTransformationMethod();
 176                 if (method == null){
 177                         return null;
 178                 }
 179                 try {
 180                         userString = (String)state.getConfig().getUserTransformationMethod().invoke(null, userString);
 181                 } catch (Exception e) {
 182                         logger.warn("Error when trying to transform userString " +  userString + ". No transformation done.");
 183                 }
 184                 User user = state.getUser(userString);
 185                 return user;
 186         }
 187
 188         private User makeNewUser(String userString, STATE state){
 189                 String pwd = getPassword();
 190                 User user = User.NewInstance(userString, pwd);
 191                 state.putUser(userString, user);
 192                 getUserService().save(user);
 193                 logger.info("Added new user: " + userString);
 194                 return user;
 195         }
 196
 197         private String getPassword(){
 198                 String result = UUID.randomUUID().toString();
 199                 return result;
 200         }
 201
 202         protected boolean resultSetHasColumn(ResultSet rs, String columnName){
 203                 try {
 204                         ResultSetMetaData metaData = rs.getMetaData();
 205                         for (int i = 0; i < metaData.getColumnCount(); i++){
 206                                 if (metaData.getColumnName(i + 1).equalsIgnoreCase(columnName)){
 207                                         return true;
 208                                 }
 209                         }
 210                         return false;
 211                 } catch (SQLException e) {
 212             logger.warn("Exception in resultSetHasColumn");
 213             return false;
 214                 }
 215         }
 216
 217
 218         protected boolean checkSqlServerColumnExists(Source source, String tableName, String columnName){
 219                 String strQuery = "SELECT  Count(t.id) as n " +
 220                                 " FROM sysobjects AS t " +
 221                                 " INNER JOIN syscolumns AS c ON t.id = c.id " +
 222                                 " WHERE (t.xtype = 'U') AND " +
 223                                 " (t.name = '" + tableName + "') AND " +
 224                                 " (c.name = '" + columnName + "')";
 225                 ResultSet rs = source.getResultSet(strQuery) ;
 226                 int n;
 227                 try {
 228                         rs.next();
 229                         n = rs.getInt("n");
 230                         return n>0;
 231                 } catch (SQLException e) {
 232                         e.printStackTrace();
 233                         return false;
 234                 }
 235
 236         }
 237
 238
 239
 240         /**
 241          * Returns a map that holds all values of a ResultSet. This is needed if a value needs to
 242          * be accessed twice
 243          * @param rs
 244          * @return
 245          * @throws SQLException
 246          */
 247         protected Map<String, Object> getValueMap(ResultSet rs) throws SQLException{
 248                 try{
 249                         Map<String, Object> valueMap = new HashMap<String, Object>();
 250                         int colCount = rs.getMetaData().getColumnCount();
 251                         for (int c = 0; c < colCount ; c++){
 252                                 Object value = rs.getObject(c+1);
 253                                 String label = rs.getMetaData().getColumnLabel(c+1).toLowerCase();
 254                                 if (value != null && ! CdmUtils.Nz(value.toString()).trim().equals("")){
 255                                         valueMap.put(label, value);
 256                                 }
 257                         }
 258                         return valueMap;
 259                 }catch(SQLException e){
 260                         throw e;
 261                 }
 262         }
 263
 264         /**
 265          * Reads a foreign key field from the result set and adds its value to the idSet.
 266          * @param rs
 267          * @param teamIdSet
 268          * @throws SQLException
 269          */
 270         protected void handleForeignKey(ResultSet rs, Set<String> idSet, String attributeName)
 271                         throws SQLException {
 272                 Object idObj = rs.getObject(attributeName);
 273                 if (idObj != null){
 274                         String id  = String.valueOf(idObj);
 275                         idSet.add(id);
 276                 }
 277         }
 278
 279         /**
 280          * Returns true if i is a multiple of recordsPerTransaction
 281          * @param i
 282          * @param recordsPerTransaction
 283          * @return
 284          */
 285         protected boolean loopNeedsHandling(int i, int recordsPerLoop) {
 286                 startTransaction();
 287                 return (i % recordsPerLoop) == 0;
 288                 }
 289
 290         protected void doLogPerLoop(int count, int recordsPerLog, String pluralString){
 291                 if ((count % recordsPerLog ) == 0 && count!= 0 ){ logger.info(pluralString + " handled: " + (count));}
 292         }
 293
 294
 295
 296
 297
 298
 299 }