app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/RoteListeDbImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.redlist;
  11
  12 import java.sql.ResultSet;
  13 import java.sql.SQLException;
  14 import java.sql.Timestamp;
  15 import java.util.HashMap;
  16 import java.util.HashSet;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22 import org.joda.time.DateTime;
  23
  24 import eu.etaxonomy.cdm.common.CdmUtils;
  25 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  26 import eu.etaxonomy.cdm.io.common.ICdmIO;
  27 import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;
  28 import eu.etaxonomy.cdm.io.common.IPartitionedIO;
  29 import eu.etaxonomy.cdm.io.common.ImportHelper;
  30 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  31 import eu.etaxonomy.cdm.io.common.Source;
  32 import eu.etaxonomy.cdm.io.common.mapping.DbImportMapping;
  33 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  34 import eu.etaxonomy.cdm.model.common.Annotation;
  35 import eu.etaxonomy.cdm.model.common.AnnotationType;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.ExtensionType;
  38 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  39 import eu.etaxonomy.cdm.model.common.Language;
  40 import eu.etaxonomy.cdm.model.common.MarkerType;
  41 import eu.etaxonomy.cdm.model.common.User;
  42
  43 /**
  44  * @author a.mueller
  45  * @created 27.08.2012
  46  * @version 1.0
  47  */
  48 public abstract class RoteListeDbImportBase<CDM_BASE extends CdmBase> extends CdmImportBase<RoteListeDbImportConfigurator, RoteListeDbImportState> implements ICdmIO<RoteListeDbImportState>, IPartitionedIO<RoteListeDbImportState> {
  49         private static final Logger logger = Logger.getLogger(RoteListeDbImportBase.class);
  50
  51
  52         private String pluralString;
  53         private String dbTableName;
  54         //TODO needed?
  55         private Class cdmTargetClass;
  56
  57
  58
  59
  60         /**
  61          * @param dbTableName
  62          * @param dbTableName2
  63          */
  64         public RoteListeDbImportBase(String pluralString, String dbTableName, Class cdmTargetClass) {
  65                 this.pluralString = pluralString;
  66                 this.dbTableName = dbTableName;
  67                 this.cdmTargetClass = cdmTargetClass;
  68         }
  69
  70         protected void doInvoke(RoteListeDbImportState state){
  71                 logger.info("start make " + getPluralString() + " ...");
  72                 RoteListeDbImportConfigurator config = state.getConfig();
  73                 Source source = config.getSource();
  74
  75                 String strIdQuery = getIdQuery();
  76                 String strRecordQuery = getRecordQuery(config);
  77
  78                 int recordsPerTransaction = config.getRecordsPerTransaction();
  79                 try{
  80                         ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery, strRecordQuery, recordsPerTransaction);
  81                         while (partitioner.nextPartition()){
  82                                 partitioner.doPartition(this, state);
  83                         }
  84                 } catch (SQLException e) {
  85                         logger.error("SQLException:" +  e);
  86                         state.setUnsuccessfull();
  87                         return;
  88                 }
  89
  90                 logger.info("end make " + getPluralString() + " ... " + getSuccessString(true));
  91                 return;
  92         }
  93
  94         public boolean doPartition(ResultSetPartitioner partitioner, RoteListeDbImportState state) {
  95                 boolean success = true ;
  96                 Set objectsToSave = new HashSet();
  97
  98                 DbImportMapping<?, ?> mapping = getMapping();
  99                 mapping.initialize(state, cdmTargetClass);
 100
 101                 ResultSet rs = partitioner.getResultSet();
 102                 try{
 103                         while (rs.next()){
 104                                 success &= mapping.invoke(rs,objectsToSave);
 105                         }
 106                 } catch (SQLException e) {
 107                         logger.error("SQLException:" +  e);
 108                         return false;
 109                 }
 110
 111                 partitioner.startDoSave();
 112                 getCommonService().save(objectsToSave);
 113                 return success;
 114         }
 115
 116
 117
 118         /**
 119          * @return
 120          */
 121         protected abstract DbImportMapping<?, ?> getMapping();
 122
 123         /**
 124          * @return
 125          */
 126         protected abstract String getRecordQuery(RoteListeDbImportConfigurator config);
 127
 128         /**
 129          * @return
 130          */
 131         protected String getIdQuery(){
 132                 String result = " SELECT id FROM " + getTableName();
 133                 return result;
 134         }
 135
 136         /* (non-Javadoc)
 137          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getPluralString()
 138          */
 139         public String getPluralString(){
 140                 return pluralString;
 141         }
 142
 143         /**
 144          * @return
 145          */
 146         protected String getTableName(){
 147                 return this.dbTableName;
 148         }
 149
 150         protected boolean doIdCreatedUpdatedNotes(RoteListeDbImportState state, IdentifiableEntity identifiableEntity, ResultSet rs, long id, String namespace)
 151                         throws SQLException{
 152                 boolean success = true;
 153                 //id
 154                 success &= ImportHelper.setOriginalSource(identifiableEntity, state.getConfig().getSourceReference(), id, namespace);
 155                 //createdUpdateNotes
 156                 success &= doCreatedUpdatedNotes(state, identifiableEntity, rs, namespace);
 157                 return success;
 158         }
 159
 160
 161         protected boolean doCreatedUpdatedNotes(RoteListeDbImportState state, AnnotatableEntity annotatableEntity, ResultSet rs, String namespace)
 162                         throws SQLException{
 163
 164                 RoteListeDbImportConfigurator config = state.getConfig();
 165                 Object createdWhen = rs.getObject("Created_When");
 166                 String createdWho = rs.getString("Created_Who");
 167                 Object updatedWhen = null;
 168                 String updatedWho = null;
 169                 try {
 170                         updatedWhen = rs.getObject("Updated_When");
 171                         updatedWho = rs.getString("Updated_who");
 172                 } catch (SQLException e) {
 173                         //Table "Name" has no updated when/who
 174                 }
 175                 String notes = rs.getString("notes");
 176
 177                 boolean success  = true;
 178
 179                 //Created When, Who, Updated When Who
 180                 if (config.getEditor() == null || config.getEditor().equals(EDITOR.NO_EDITORS)){
 181                         //do nothing
 182                 }else if (config.getEditor().equals(EDITOR.EDITOR_AS_ANNOTATION)){
 183                         String createdAnnotationString = "Berlin Model record was created By: " + String.valueOf(createdWho) + " (" + String.valueOf(createdWhen) + ") ";
 184                         if (updatedWhen != null && updatedWho != null){
 185                                 createdAnnotationString += " and updated By: " + String.valueOf(updatedWho) + " (" + String.valueOf(updatedWhen) + ")";
 186                         }
 187                         Annotation annotation = Annotation.NewInstance(createdAnnotationString, Language.DEFAULT());
 188                         annotation.setCommentator(config.getCommentator());
 189                         annotation.setAnnotationType(AnnotationType.TECHNICAL());
 190                         annotatableEntity.addAnnotation(annotation);
 191                 }else if (config.getEditor().equals(EDITOR.EDITOR_AS_EDITOR)){
 192                         User creator = getUser(createdWho, state);
 193                         User updator = getUser(updatedWho, state);
 194                         DateTime created = getDateTime(createdWhen);
 195                         DateTime updated = getDateTime(updatedWhen);
 196                         annotatableEntity.setCreatedBy(creator);
 197                         annotatableEntity.setUpdatedBy(updator);
 198                         annotatableEntity.setCreated(created);
 199                         annotatableEntity.setUpdated(updated);
 200                 }else {
 201                         logger.warn("Editor type not yet implemented: " + config.getEditor());
 202                 }
 203
 204
 205                 //notes
 206                 if (CdmUtils.isNotEmpty(notes)){
 207                         String notesString = String.valueOf(notes);
 208                         if (notesString.length() > 65530 ){
 209                                 notesString = notesString.substring(0, 65530) + "...";
 210                                 logger.warn("Notes string is longer than 65530 and was truncated: " + annotatableEntity);
 211                         }
 212                         Annotation notesAnnotation = Annotation.NewInstance(notesString, null);
 213                         //notesAnnotation.setAnnotationType(AnnotationType.EDITORIAL());
 214                         //notes.setCommentator(bmiConfig.getCommentator());
 215                         annotatableEntity.addAnnotation(notesAnnotation);
 216                 }
 217                 return success;
 218         }
 219
 220
 221         private User getUser(String createdWho, RoteListeDbImportState state) {
 222                 //not relevant here, for users see ERMS import
 223                 return null;
 224         }
 225
 226         private DateTime getDateTime(Object timeString){
 227                 if (timeString == null){
 228                         return null;
 229                 }
 230                 DateTime dateTime = null;
 231                 if (timeString instanceof Timestamp){
 232                         Timestamp timestamp = (Timestamp)timeString;
 233                         dateTime = new DateTime(timestamp);
 234                 }else{
 235                         logger.warn("time ("+timeString+") is not a timestamp. Datetime set to current date. ");
 236                         dateTime = new DateTime();
 237                 }
 238                 return dateTime;
 239         }
 240
 241
 242         /**
 243          * Returns a map that holds all values of a ResultSet. This is needed if a value needs to
 244          * be accessed twice
 245          * @param rs
 246          * @return
 247          * @throws SQLException
 248          */
 249         protected Map<String, Object> getValueMap(ResultSet rs) throws SQLException{
 250                 try{
 251                         Map<String, Object> valueMap = new HashMap<String, Object>();
 252                         int colCount = rs.getMetaData().getColumnCount();
 253                         for (int c = 0; c < colCount ; c++){
 254                                 Object value = rs.getObject(c+1);
 255                                 String label = rs.getMetaData().getColumnLabel(c+1).toLowerCase();
 256                                 if (value != null && ! CdmUtils.Nz(value.toString()).trim().equals("")){
 257                                         valueMap.put(label, value);
 258                                 }
 259                         }
 260                         return valueMap;
 261                 }catch(SQLException e){
 262                         throw e;
 263                 }
 264         }
 265
 266         protected ExtensionType getExtensionType(UUID uuid, String label, String text, String labelAbbrev){
 267                 ExtensionType extensionType = (ExtensionType)getTermService().find(uuid);
 268                 if (extensionType == null){
 269                         extensionType = ExtensionType.NewInstance(text, label, labelAbbrev);
 270                         extensionType.setUuid(uuid);
 271                         getTermService().save(extensionType);
 272                 }
 273                 return extensionType;
 274         }
 275
 276         protected MarkerType getMarkerType(UUID uuid, String label, String text, String labelAbbrev){
 277                 MarkerType markerType = (MarkerType)getTermService().find(uuid);
 278                 if (markerType == null){
 279                         markerType = MarkerType.NewInstance(label, text, labelAbbrev);
 280                         markerType.setUuid(uuid);
 281                         getTermService().save(markerType);
 282                 }
 283                 return markerType;
 284         }
 285
 286
 287         /**
 288          * Reads a foreign key field from the result set and adds its value to the idSet.
 289          * @param rs
 290          * @param teamIdSet
 291          * @throws SQLException
 292          */
 293         protected void handleForeignKey(ResultSet rs, Set<String> idSet, String attributeName)
 294                         throws SQLException {
 295                 Object idObj = rs.getObject(attributeName);
 296                 if (idObj != null){
 297                         String id  = String.valueOf(idObj);
 298                         idSet.add(id);
 299                 }
 300         }
 301
 302         /**
 303          * Returns true if i is a multiple of recordsPerTransaction
 304          * @param i
 305          * @param recordsPerTransaction
 306          * @return
 307          */
 308         protected boolean loopNeedsHandling(int i, int recordsPerLoop) {
 309                 startTransaction();
 310                 return (i % recordsPerLoop) == 0;
 311         }
 312
 313         protected void doLogPerLoop(int count, int recordsPerLog, String pluralString){
 314                 if ((count % recordsPerLog ) == 0 && count!= 0 ){ logger.info(pluralString + " handled: " + (count));}
 315         }
 316
 317
 318
 319
 320 }