app-import/src/main/java/eu/etaxonomy/cdm/io/globis/GlobisImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.globis;
  11
  12 import java.lang.reflect.Method;
  13 import java.sql.ResultSet;
  14 import java.sql.ResultSetMetaData;
  15 import java.sql.SQLException;
  16 import java.sql.Timestamp;
  17 import java.util.HashMap;
  18 import java.util.HashSet;
  19 import java.util.Map;
  20 import java.util.Set;
  21 import java.util.UUID;
  22
  23 import org.apache.commons.lang.StringUtils;
  24 import org.apache.log4j.Logger;
  25 import org.joda.time.DateTime;
  26
  27 import eu.etaxonomy.cdm.common.CdmUtils;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.ICdmIO;
  30 import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;
  31 import eu.etaxonomy.cdm.io.common.IPartitionedIO;
  32 import eu.etaxonomy.cdm.io.common.ImportHelper;
  33 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  34 import eu.etaxonomy.cdm.io.common.Source;
  35 import eu.etaxonomy.cdm.io.common.mapping.DbImportMapping;
  36 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  37 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  38 import eu.etaxonomy.cdm.model.common.Annotation;
  39 import eu.etaxonomy.cdm.model.common.AnnotationType;
  40 import eu.etaxonomy.cdm.model.common.CdmBase;
  41 import eu.etaxonomy.cdm.model.common.ExtensionType;
  42 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  43 import eu.etaxonomy.cdm.model.common.Language;
  44 import eu.etaxonomy.cdm.model.common.MarkerType;
  45 import eu.etaxonomy.cdm.model.common.User;
  46 import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
  47 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  48 import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;
  49 import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;
  50 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  51
  52 /**
  53  * @author a.mueller
  54  * @created 20.03.2008
  55  * @version 1.0
  56  */
  57 public abstract class GlobisImportBase<CDM_BASE extends CdmBase> extends CdmImportBase<GlobisImportConfigurator, GlobisImportState> implements ICdmIO<GlobisImportState>, IPartitionedIO<GlobisImportState> {
  58         private static final Logger logger = Logger.getLogger(GlobisImportBase.class);
  59
  60         public static final UUID ID_IN_SOURCE_EXT_UUID = UUID.fromString("23dac094-e793-40a4-bad9-649fc4fcfd44");
  61
  62         //NAMESPACES
  63
  64         protected static final String REFERENCE_NAMESPACE = "Literatur";
  65         protected static final String TAXON_NAMESPACE = "current_species";
  66
  67
  68         private String pluralString;
  69         private String dbTableName;
  70         //TODO needed?
  71         private Class cdmTargetClass;
  72
  73         private INonViralNameParser parser = NonViralNameParserImpl.NewInstance();
  74
  75
  76         /**
  77          * @param dbTableName
  78          * @param dbTableName2
  79          */
  80         public GlobisImportBase(String pluralString, String dbTableName, Class cdmTargetClass) {
  81                 this.pluralString = pluralString;
  82                 this.dbTableName = dbTableName;
  83                 this.cdmTargetClass = cdmTargetClass;
  84         }
  85
  86         protected void doInvoke(GlobisImportState state){
  87                 logger.info("start make " + getPluralString() + " ...");
  88                 GlobisImportConfigurator config = state.getConfig();
  89                 Source source = config.getSource();
  90
  91                 String strIdQuery = getIdQuery();
  92                 String strRecordQuery = getRecordQuery(config);
  93
  94                 int recordsPerTransaction = config.getRecordsPerTransaction();
  95                 try{
  96                         ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery, strRecordQuery, recordsPerTransaction);
  97                         while (partitioner.nextPartition()){
  98                                 partitioner.doPartition(this, state);
  99                         }
 100                 } catch (SQLException e) {
 101                         logger.error("SQLException:" +  e);
 102                         state.setUnsuccessfull();
 103                 }
 104
 105                 logger.info("end make " + getPluralString() + " ... " + getSuccessString(true));
 106                 return;
 107         }
 108
 109         /**
 110          * @param authorAndYear
 111          * @param zooName
 112          */
 113         protected void handleAuthorAndYear(String authorAndYear, ZoologicalName zooName) {
 114                 if (isBlank(authorAndYear)){
 115                         return;
 116                 }
 117                 try {
 118                         String doubtfulAuthorAndYear = null;
 119                         if(authorAndYear.matches(".+\\,\\s\\[\\d{4}\\].*")){
 120                                 doubtfulAuthorAndYear = authorAndYear;
 121                                 authorAndYear = authorAndYear.replace("[", "").replace("]", "");
 122                         }
 123                         if (authorAndYear.contains("?")){
 124                                 authorAndYear = authorAndYear.replace("H?bner", "H\u00fcbner");
 125                                 authorAndYear = authorAndYear.replace("Oberth?r", "Oberth\u00fcr");
 126                         }
 127
 128                         parser.parseAuthors(zooName, authorAndYear);
 129                         if (doubtfulAuthorAndYear != null){
 130                                 zooName.setAuthorshipCache(doubtfulAuthorAndYear, true);
 131                         }
 132
 133                 } catch (StringNotParsableException e) {
 134                         logger.warn("Author could not be parsed: " + authorAndYear);
 135                         zooName.setAuthorshipCache(authorAndYear, true);
 136                 }
 137         }
 138
 139
 140         /**
 141          * @param state
 142          * @param countryStr
 143          * @return
 144          */
 145         protected WaterbodyOrCountry getCountry(GlobisImportState state, String countryStr) {
 146                 WaterbodyOrCountry country = WaterbodyOrCountry.getWaterbodyOrCountryByLabel(countryStr);
 147                 if (country == null){
 148                         try {
 149                                 country = (WaterbodyOrCountry)state.getTransformer().getNamedAreaByKey(countryStr);
 150                         } catch (UndefinedTransformerMethodException e) {
 151                                 e.printStackTrace();
 152                         }
 153                 }
 154                 return country;
 155         }
 156
 157
 158
 159         public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) {
 160                 boolean success = true ;
 161                 Set objectsToSave = new HashSet();
 162
 163                 DbImportMapping<?, ?> mapping = getMapping();
 164                 mapping.initialize(state, cdmTargetClass);
 165
 166                 ResultSet rs = partitioner.getResultSet();
 167                 try{
 168                         while (rs.next()){
 169                                 success &= mapping.invoke(rs,objectsToSave);
 170                         }
 171                 } catch (SQLException e) {
 172                         logger.error("SQLException:" +  e);
 173                         return false;
 174                 }
 175
 176                 partitioner.startDoSave();
 177                 getCommonService().save(objectsToSave);
 178                 return success;
 179         }
 180
 181
 182
 183         /**
 184          * @return
 185          */
 186         protected /*abstract*/ DbImportMapping<?, ?> getMapping(){
 187                 return null;
 188         }
 189
 190         /**
 191          * @return
 192          */
 193         protected abstract String getRecordQuery(GlobisImportConfigurator config);
 194
 195         /**
 196          * @return
 197          */
 198         protected String getIdQuery(){
 199                 String result = " SELECT id FROM " + getTableName();
 200                 return result;
 201         }
 202
 203         /* (non-Javadoc)
 204          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getPluralString()
 205          */
 206         public String getPluralString(){
 207                 return pluralString;
 208         }
 209
 210         /**
 211          * @return
 212          */
 213         protected String getTableName(){
 214                 return this.dbTableName;
 215         }
 216
 217         protected boolean doIdCreatedUpdatedNotes(GlobisImportState state, IdentifiableEntity identifiableEntity, ResultSet rs, long id, String namespace)
 218                         throws SQLException{
 219                 boolean success = true;
 220                 //id
 221                 success &= ImportHelper.setOriginalSource(identifiableEntity, state.getConfig().getSourceReference(), id, namespace);
 222                 //createdUpdateNotes
 223                 success &= doCreatedUpdatedNotes(state, identifiableEntity, rs, namespace);
 224                 return success;
 225         }
 226
 227
 228         protected boolean doCreatedUpdatedNotes(GlobisImportState state, AnnotatableEntity annotatableEntity, ResultSet rs, String namespace)
 229                         throws SQLException{
 230
 231                 GlobisImportConfigurator config = state.getConfig();
 232                 Object createdWhen = rs.getObject("Created_When");
 233                 String createdWho = rs.getString("Created_Who");
 234                 Object updatedWhen = null;
 235                 String updatedWho = null;
 236                 try {
 237                         updatedWhen = rs.getObject("Updated_When");
 238                         updatedWho = rs.getString("Updated_who");
 239                 } catch (SQLException e) {
 240                         //Table "Name" has no updated when/who
 241                 }
 242                 String notes = rs.getString("notes");
 243
 244                 boolean success  = true;
 245
 246                 //Created When, Who, Updated When Who
 247                 if (config.getEditor() == null || config.getEditor().equals(EDITOR.NO_EDITORS)){
 248                         //do nothing
 249                 }else if (config.getEditor().equals(EDITOR.EDITOR_AS_ANNOTATION)){
 250                         String createdAnnotationString = "Berlin Model record was created By: " + String.valueOf(createdWho) + " (" + String.valueOf(createdWhen) + ") ";
 251                         if (updatedWhen != null && updatedWho != null){
 252                                 createdAnnotationString += " and updated By: " + String.valueOf(updatedWho) + " (" + String.valueOf(updatedWhen) + ")";
 253                         }
 254                         Annotation annotation = Annotation.NewInstance(createdAnnotationString, Language.DEFAULT());
 255                         annotation.setCommentator(config.getCommentator());
 256                         annotation.setAnnotationType(AnnotationType.TECHNICAL());
 257                         annotatableEntity.addAnnotation(annotation);
 258                 }else if (config.getEditor().equals(EDITOR.EDITOR_AS_EDITOR)){
 259                         User creator = getUser(createdWho, state);
 260                         User updator = getUser(updatedWho, state);
 261                         DateTime created = getDateTime(createdWhen);
 262                         DateTime updated = getDateTime(updatedWhen);
 263                         annotatableEntity.setCreatedBy(creator);
 264                         annotatableEntity.setUpdatedBy(updator);
 265                         annotatableEntity.setCreated(created);
 266                         annotatableEntity.setUpdated(updated);
 267                 }else {
 268                         logger.warn("Editor type not yet implemented: " + config.getEditor());
 269                 }
 270
 271
 272                 //notes
 273                 if (StringUtils.isNotBlank(notes)){
 274                         String notesString = String.valueOf(notes);
 275                         if (notesString.length() > 65530 ){
 276                                 notesString = notesString.substring(0, 65530) + "...";
 277                                 logger.warn("Notes string is longer than 65530 and was truncated: " + annotatableEntity);
 278                         }
 279                         Annotation notesAnnotation = Annotation.NewInstance(notesString, null);
 280                         //notesAnnotation.setAnnotationType(AnnotationType.EDITORIAL());
 281                         //notes.setCommentator(bmiConfig.getCommentator());
 282                         annotatableEntity.addAnnotation(notesAnnotation);
 283                 }
 284                 return success;
 285         }
 286
 287         private User getUser(String userString, GlobisImportState state){
 288                 if (StringUtils.isBlank(userString)){
 289                         return null;
 290                 }
 291                 userString = userString.trim();
 292
 293                 User user = state.getUser(userString);
 294                 if (user == null){
 295                         user = getTransformedUser(userString,state);
 296                 }
 297                 if (user == null){
 298                         user = makeNewUser(userString, state);
 299                 }
 300                 if (user == null){
 301                         logger.warn("User is null");
 302                 }
 303                 return user;
 304         }
 305
 306         private User getTransformedUser(String userString, GlobisImportState state){
 307                 Method method = state.getConfig().getUserTransformationMethod();
 308                 if (method == null){
 309                         return null;
 310                 }
 311                 try {
 312                         userString = (String)state.getConfig().getUserTransformationMethod().invoke(null, userString);
 313                 } catch (Exception e) {
 314                         logger.warn("Error when trying to transform userString " +  userString + ". No transformation done.");
 315                 }
 316                 User user = state.getUser(userString);
 317                 return user;
 318         }
 319
 320         private User makeNewUser(String userString, GlobisImportState state){
 321                 String pwd = getPassword();
 322                 User user = User.NewInstance(userString, pwd);
 323                 state.putUser(userString, user);
 324                 getUserService().save(user);
 325                 logger.info("Added new user: " + userString);
 326                 return user;
 327         }
 328
 329         private String getPassword(){
 330                 String result = UUID.randomUUID().toString();
 331                 return result;
 332         }
 333
 334         private DateTime getDateTime(Object timeString){
 335                 if (timeString == null){
 336                         return null;
 337                 }
 338                 DateTime dateTime = null;
 339                 if (timeString instanceof Timestamp){
 340                         Timestamp timestamp = (Timestamp)timeString;
 341                         dateTime = new DateTime(timestamp);
 342                 }else{
 343                         logger.warn("time ("+timeString+") is not a timestamp. Datetime set to current date. ");
 344                         dateTime = new DateTime();
 345                 }
 346                 return dateTime;
 347         }
 348
 349
 350
 351         /**
 352          * Reads a foreign key field from the result set and adds its value to the idSet.
 353          * @param rs
 354          * @param teamIdSet
 355          * @throws SQLException
 356          */
 357         protected void handleForeignKey(ResultSet rs, Set<String> idSet, String attributeName)
 358                         throws SQLException {
 359                 Object idObj = rs.getObject(attributeName);
 360                 if (idObj != null){
 361                         String id  = String.valueOf(idObj);
 362                         idSet.add(id);
 363                 }
 364         }
 365
 366
 367
 368
 369         /**
 370          * Returns true if i is a multiple of recordsPerTransaction
 371          * @param i
 372          * @param recordsPerTransaction
 373          * @return
 374          */
 375         protected boolean loopNeedsHandling(int i, int recordsPerLoop) {
 376                 startTransaction();
 377                 return (i % recordsPerLoop) == 0;
 378         }
 379
 380         protected void doLogPerLoop(int count, int recordsPerLog, String pluralString){
 381                 if ((count % recordsPerLog ) == 0 && count!= 0 ){ logger.info(pluralString + " handled: " + (count));}
 382         }
 383
 384
 385
 386
 387 }