app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_ACCEPTED;
  13 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_PARTIAL_SYN;
  14 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_PRO_PARTE_SYN;
  15 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_SYNONYM;
  16 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_UNRESOLVED;
  17
  18 import java.lang.reflect.Method;
  19 import java.sql.ResultSet;
  20 import java.sql.SQLException;
  21 import java.util.HashMap;
  22 import java.util.HashSet;
  23 import java.util.Map;
  24 import java.util.Set;
  25 import java.util.UUID;
  26
  27 import org.apache.commons.lang.StringUtils;
  28 import org.apache.log4j.Logger;
  29 import org.springframework.stereotype.Component;
  30
  31 import eu.etaxonomy.cdm.database.update.DatabaseTypeNotSupportedException;
  32 import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
  33 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelTaxonImportValidator;
  34 import eu.etaxonomy.cdm.io.common.IOValidator;
  35 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  36 import eu.etaxonomy.cdm.model.agent.Person;
  37 import eu.etaxonomy.cdm.model.common.CdmBase;
  38 import eu.etaxonomy.cdm.model.common.Extension;
  39 import eu.etaxonomy.cdm.model.common.ExtensionType;
  40 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  41 import eu.etaxonomy.cdm.model.reference.Reference;
  42 import eu.etaxonomy.cdm.model.taxon.Synonym;
  43 import eu.etaxonomy.cdm.model.taxon.Taxon;
  44 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  45
  46
  47 /**
  48  * @author a.mueller
  49  * @created 20.03.2008
  50  * @version 1.0
  51  */
  52 @Component
  53 public class BerlinModelTaxonImport  extends BerlinModelImportBase {
  54         private static final Logger logger = Logger.getLogger(BerlinModelTaxonImport.class);
  55
  56         public static final String NAMESPACE = "Taxon";
  57
  58         private static final String pluralString = "Taxa";
  59         private static final String dbTableName = "PTaxon";
  60
  61         /**
  62          * How should the publish flag in table PTaxon be interpreted
  63          * NO_MARKER: No marker is set
  64          * ONLY_FALSE:
  65          */
  66         public enum PublishMarkerChooser{
  67                 NO_MARKER,
  68                 ONLY_FALSE,
  69                 ONLY_TRUE,
  70                 ALL;
  71
  72                 boolean doMark(boolean value){
  73                         if (value == true){
  74                                 return this == ALL || this == ONLY_TRUE;
  75                         }else{
  76                                 return this == ALL || this == ONLY_FALSE;
  77                         }
  78                 }
  79         }
  80
  81         public BerlinModelTaxonImport(){
  82                 super(dbTableName, pluralString);
  83         }
  84
  85         @Override
  86         protected String getIdQuery(BerlinModelImportState state) {
  87                 String sqlSelect = " SELECT RIdentifier";
  88                 String taxonTable = state.getConfig().getTaxonTable();
  89                 String sqlFrom = String.format(" FROM %s ", taxonTable);
  90                 String sqlWhere = "";
  91
  92                 String sql = sqlSelect + " " + sqlFrom + " " + sqlWhere ;
  93                 return sql;
  94         }
  95
  96         @Override
  97         protected String getRecordQuery(BerlinModelImportConfigurator config) {
  98                 String sqlSelect = " SELECT pt.*  ";
  99                 String sqlFrom = " FROM PTaxon pt ";
 100                 if (isEuroMed(config) ){
 101                         sqlFrom = " FROM PTaxon AS pt " +
 102                                                         " INNER JOIN v_cdm_exp_taxaAll AS em ON pt.RIdentifier = em.RIdentifier " +
 103                                                         " LEFT OUTER JOIN Reference r ON pt.LastScrutinyFk = r.RefId ";
 104                         sqlSelect += " , em.MA, r.RefCache as LastScrutiny ";
 105                 }
 106
 107
 108                 String sqlWhere = " WHERE ( pt.RIdentifier IN (" + ID_LIST_TOKEN + ") )";
 109
 110                 String strRecordQuery =sqlSelect + " " + sqlFrom + " " + sqlWhere ;
 111 //                      " SELECT * " +
 112 //                      " FROM PTaxon " + state.getConfig().getTaxonTable();
 113 //                      " WHERE ( RIdentifier IN (" + ID_LIST_TOKEN + ") )";
 114                 return strRecordQuery;
 115         }
 116
 117         private boolean isEuroMed(BerlinModelImportConfigurator config) {
 118                 return config.getTaxonTable().trim().equals("v_cdm_exp_taxaAll");
 119         }
 120
 121         @Override
 122         protected boolean doCheck(BerlinModelImportState state){
 123                 IOValidator<BerlinModelImportState> validator = new BerlinModelTaxonImportValidator();
 124                 return validator.validate(state);
 125         }
 126
 127         @Override
 128         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 129                 boolean success = true ;
 130
 131                 BerlinModelImportConfigurator config = state.getConfig();
 132                 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
 133                 Map<String, TaxonNameBase> taxonNameMap = partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);
 134                 Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
 135
 136                 ResultSet rs = partitioner.getResultSet();
 137                 try{
 138                         boolean publishFlagExists = state.getConfig().getSource().checkColumnExists("PTaxon", "PublishFlag");
 139                         boolean isEuroMed = isEuroMed(state.getConfig());
 140                         while (rs.next()){
 141
 142                         //      if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("PTaxa handled: " + (i-1));}
 143
 144                                 //create TaxonName element
 145                                 int taxonId = rs.getInt("RIdentifier");
 146                                 int statusFk = rs.getInt("statusFk");
 147
 148                                 int nameFk = rs.getInt("PTNameFk");
 149                                 int refFkInt = rs.getInt("PTRefFk");
 150                                 String doubtful = rs.getString("DoubtfulFlag");
 151                                 String uuid = null;
 152                                 if (resultSetHasColumn(rs,"UUID")){
 153                                         uuid = rs.getString("UUID");
 154                                 }
 155
 156                                 TaxonNameBase<?,?> taxonName = null;
 157                                 taxonName  = taxonNameMap.get(String.valueOf(nameFk));
 158
 159                                 Reference reference = null;
 160                                 String refFk = String.valueOf(refFkInt);
 161                                 reference = refMap.get(refFk);
 162
 163                                 if(! config.isIgnoreNull()){
 164                                         if (taxonName == null ){
 165                                                 logger.warn("TaxonName belonging to taxon (RIdentifier = " + taxonId + ") could not be found in store. Taxon will not be imported");
 166                                                 success = false;
 167                                                 continue; //next taxon
 168                                         }else if (reference == null ){
 169                                                 logger.warn("Reference belonging to taxon could not be found in store. Taxon will not be imported");
 170                                                 success = false;
 171                                                 continue; //next taxon
 172                                         }
 173                                 }
 174                                 TaxonBase<?> taxonBase;
 175                                 Synonym synonym;
 176                                 Taxon taxon;
 177                                 try {
 178                                         logger.debug(statusFk);
 179                                         if (statusFk == T_STATUS_ACCEPTED || statusFk == T_STATUS_UNRESOLVED ){
 180                                                 taxon = Taxon.NewInstance(taxonName, reference);
 181                                                 taxonBase = taxon;
 182                                                 if (statusFk == T_STATUS_UNRESOLVED){
 183                                                         taxon.setTaxonStatusUnknown(true);
 184                                                 }
 185                                         }else if (statusFk == T_STATUS_SYNONYM || statusFk == T_STATUS_PRO_PARTE_SYN || statusFk == T_STATUS_PARTIAL_SYN){
 186                                                 synonym = Synonym.NewInstance(taxonName, reference);
 187                                                 taxonBase = synonym;
 188                                                 if (statusFk == T_STATUS_PRO_PARTE_SYN){
 189                                                     synonym.setProParte(true);
 190                                                 }
 191                                                 if (statusFk == T_STATUS_PARTIAL_SYN){
 192                                                         synonym.setPartial(true);
 193                                                 }
 194                                         }else{
 195                                                 logger.warn("TaxonStatus " + statusFk + " not yet implemented. Taxon (RIdentifier = " + taxonId + ") left out.");
 196                                                 success = false;
 197                                                 continue;
 198                                         }
 199                                         if (uuid != null){
 200                                                 taxonBase.setUuid(UUID.fromString(uuid));
 201                                         }
 202
 203                                         //doubtful
 204                                         if (doubtful.equals("a")){
 205                                                 taxonBase.setDoubtful(false);
 206                                         }else if(doubtful.equals("d")){
 207                                                 taxonBase.setDoubtful(true);
 208                                         }else if(doubtful.equals("i")){
 209                                                 taxonBase.setDoubtful(false);
 210                                                 logger.warn("Doubtful = i (inactivated) does not exist in CDM. Doubtful set to false. RIdentifier: " + taxonId);
 211                                         }
 212
 213                                         //detail
 214                                         String detail = rs.getString("Detail");
 215                                         if (StringUtils.isNotBlank(detail)){
 216                                                 ExtensionType detailExtensionType = getExtensionType(state, BerlinModelTransformer.DETAIL_EXT_UUID, "micro reference","micro reference","micro ref.");
 217                                                 Extension.NewInstance(taxonBase, detail, detailExtensionType);
 218                                         }
 219                                         //idInSource
 220                                         String idInSource = rs.getString("IdInSource");
 221                                         if (StringUtils.isNotBlank(idInSource)){
 222                                                 ExtensionType detailExtensionType = getExtensionType(state, BerlinModelTransformer.ID_IN_SOURCE_EXT_UUID, "Berlin Model IdInSource","Berlin Model IdInSource","BM source id");
 223                                                 Extension.NewInstance(taxonBase, idInSource, detailExtensionType);
 224                                         }
 225                                         //namePhrase
 226                                         String namePhrase = rs.getString("NamePhrase");
 227                                         if (StringUtils.isNotBlank(namePhrase)){
 228                                                 taxonBase.setAppendedPhrase(namePhrase);
 229                                         }
 230                                         //useNameCache
 231                                         Boolean useNameCacheFlag = rs.getBoolean("UseNameCacheFlag");
 232                                         if (useNameCacheFlag){
 233                                                 taxonBase.setUseNameCache(true);
 234                                         }
 235                                         //publisheFlag
 236                                         if (publishFlagExists){
 237                                                 Boolean publishFlag = rs.getBoolean("PublishFlag");
 238                                                 Boolean misapplied = false;
 239                                                 if (isEuroMed){
 240                                                         misapplied = rs.getBoolean("MA");
 241                                                 }
 242
 243                                                 if ( ! misapplied){
 244                                                         taxonBase.setPublish(publishFlag);
 245                                                 }
 246                                         }
 247
 248                                         //
 249                                         if (resultSetHasColumn(rs,"LastScrutiny")){
 250                                                 String lastScrutiny = rs.getString("LastScrutiny");
 251                                                 ExtensionType extensionTypeSpeciesExpert = getExtensionType(state, BerlinModelTransformer.uuidSpeciesExpertName, "Species Expert", "Species Expert", "Species Expert");
 252                                                 taxonBase.addExtension(lastScrutiny, extensionTypeSpeciesExpert);
 253                                                 ExtensionType extensionTypeExpert = getExtensionType(state, BerlinModelTransformer.uuidExpertName, "Expert", "Expert for a taxonomic group", "Expert");
 254                                                 taxonBase.addExtension(lastScrutiny, extensionTypeExpert);
 255                                         }
 256
 257                                         //Notes
 258                                         doIdCreatedUpdatedNotes(state, taxonBase, rs, taxonId, NAMESPACE);
 259
 260                                         //external url
 261                                         if (config.getMakeUrlForTaxon() != null){
 262                                                 Method urlMethod = config.getMakeUrlForTaxon();
 263                                                 urlMethod.invoke(null, taxonBase, rs);
 264                                         }
 265
 266                                         partitioner.startDoSave();
 267                                         taxaToSave.add(taxonBase);
 268                                 } catch (Exception e) {
 269                                         logger.warn("An exception (" +e.getMessage()+") occurred when creating taxon with id " + taxonId + ". Taxon could not be saved.");
 270                                         success = false;
 271                                 }
 272                         }
 273                 } catch (DatabaseTypeNotSupportedException e) {
 274                         logger.error("MethodNotSupportedException:" +  e);
 275                         return false;
 276                 } catch (Exception e) {
 277                         logger.error("SQLException:" +  e);
 278                         return false;
 279                 }
 280
 281
 282                 //      logger.info( i + " names handled");
 283                 getTaxonService().save(taxaToSave);
 284                 return success;
 285         }
 286
 287         @Override
 288         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
 289                 String nameSpace;
 290                 Class<?> cdmClass;
 291                 Set<String> idSet;
 292                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 293
 294                 try{
 295                         Set<String> nameIdSet = new HashSet<String>();
 296                         Set<String> referenceIdSet = new HashSet<String>();
 297                         while (rs.next()){
 298                                 handleForeignKey(rs, nameIdSet, "PTNameFk");
 299                                 handleForeignKey(rs, referenceIdSet, "PTRefFk");
 300                         }
 301
 302                         //name map
 303                         nameSpace = BerlinModelTaxonNameImport.NAMESPACE;
 304                         cdmClass = TaxonNameBase.class;
 305                         idSet = nameIdSet;
 306                         Map<String, Person> nameMap = (Map<String, Person>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 307                         result.put(nameSpace, nameMap);
 308
 309                         //reference map
 310                         nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
 311                         cdmClass = Reference.class;
 312                         idSet = referenceIdSet;
 313                         Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 314                         result.put(nameSpace, referenceMap);
 315
 316                 } catch (SQLException e) {
 317                         throw new RuntimeException(e);
 318                 }
 319                 return result;
 320         }
 321
 322         @Override
 323         protected String getTableName() {
 324                 return dbTableName;
 325         }
 326
 327         @Override
 328         public String getPluralString() {
 329                 return pluralString;
 330         }
 331
 332         @Override
 333         protected boolean isIgnore(BerlinModelImportState state){
 334                 return ! state.getConfig().isDoTaxa();
 335         }
 336
 337
 338
 339 }