cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_ACCEPTED;
  13 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_PARTIAL_SYN;
  14 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_PRO_PARTE_SYN;
  15 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.T_STATUS_SYNONYM;
  16
  17 import java.sql.ResultSet;
  18 import java.sql.SQLException;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22 import org.springframework.stereotype.Component;
  23
  24 import eu.etaxonomy.cdm.common.CdmUtils;
  25 import eu.etaxonomy.cdm.io.common.ICdmIO;
  26 import eu.etaxonomy.cdm.io.common.ImportHelper;
  27 import eu.etaxonomy.cdm.io.common.MapWrapper;
  28 import eu.etaxonomy.cdm.io.common.Source;
  29 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  30 import eu.etaxonomy.cdm.model.common.CdmBase;
  31 import eu.etaxonomy.cdm.model.common.Extension;
  32 import eu.etaxonomy.cdm.model.common.ExtensionType;
  33 import eu.etaxonomy.cdm.model.common.Marker;
  34 import eu.etaxonomy.cdm.model.common.MarkerType;
  35 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  36 import eu.etaxonomy.cdm.model.reference.ReferenceBase;
  37 import eu.etaxonomy.cdm.model.taxon.Synonym;
  38 import eu.etaxonomy.cdm.model.taxon.Taxon;
  39 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  40
  41
  42 /**
  43  * @author a.mueller
  44  * @created 20.03.2008
  45  * @version 1.0
  46  */
  47 @Component
  48 public class BerlinModelTaxonImport  extends BerlinModelImportBase  {
  49         private static final Logger logger = Logger.getLogger(BerlinModelTaxonImport.class);
  50
  51         public static final UUID DETAIL_EXT_UUID = UUID.fromString("c3959b4f-d876-4b7a-a739-9260f4cafd1c");
  52         public static final UUID APPENDED_TITLE_PHRASE = UUID.fromString("b121f3b6-89bb-48e1-a010-7d3148d2caba");
  53         public static final UUID USE_NAME_CACHE = UUID.fromString("1b959a0d-230b-4b03-b7b6-2bd46056a22d");
  54
  55
  56         /**
  57          * How should the publish flag in table PTaxon be interpreted
  58          * NO_MARKER: No marker is set
  59          * ONLY_FALSE:
  60          */
  61         public enum PublishMarkerChooser{
  62                 NO_MARKER,
  63                 ONLY_FALSE,
  64                 ONLY_TRUE,
  65                 ALL;
  66
  67                 boolean doMark(boolean value){
  68                         if (value == true){
  69                                 return this == ALL || this == ONLY_TRUE;
  70                         }else{
  71                                 return this == ALL || this == ONLY_FALSE;
  72                         }
  73                 }
  74         }
  75
  76
  77         private int modCount = 10000;
  78
  79         public BerlinModelTaxonImport(){
  80                 super();
  81         }
  82
  83         /* (non-Javadoc)
  84          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)
  85          */
  86         @Override
  87         protected boolean doCheck(BerlinModelImportState state){
  88                 boolean result = true;
  89                 BerlinModelImportConfigurator bmiConfig = state.getConfig();
  90                 logger.warn("Checking for Taxa not yet fully implemented");
  91                 result &= checkTaxonStatus(bmiConfig);
  92                 result &= checkInactivated(bmiConfig);
  93
  94                 return result;
  95         }
  96
  97         private boolean checkTaxonStatus(BerlinModelImportConfigurator bmiConfig){
  98                 try {
  99                         boolean result = true;
 100                         Source source = bmiConfig.getSource();
 101                         String strSQL = " SELECT RelPTaxon.RelQualifierFk, RelPTaxon.relPTaxonId, PTaxon.PTNameFk, PTaxon.PTRefFk, PTaxon_1.PTNameFk AS Expr1, PTaxon.RIdentifier, PTaxon_1.RIdentifier AS Expr3, Name.FullNameCache "  +
 102                                 " FROM RelPTaxon " +
 103                                         " INNER JOIN PTaxon ON RelPTaxon.PTNameFk1 = PTaxon.PTNameFk AND RelPTaxon.PTRefFk1 = PTaxon.PTRefFk " +
 104                                         " INNER JOIN PTaxon AS PTaxon_1 ON RelPTaxon.PTNameFk2 = PTaxon_1.PTNameFk AND RelPTaxon.PTRefFk2 = PTaxon_1.PTRefFk  " +
 105                                         " INNER JOIN Name ON PTaxon.PTNameFk = Name.NameId " +
 106                                 " WHERE (dbo.PTaxon.StatusFk = 1) AND ((RelPTaxon.RelQualifierFk = 7) OR (RelPTaxon.RelQualifierFk = 6) OR (RelPTaxon.RelQualifierFk = 2)) ";
 107                         ResultSet rs = source.getResultSet(strSQL);
 108                         boolean firstRow = true;
 109                         int i = 0;
 110                         while (rs.next()){
 111                                 i++;
 112                                 if (firstRow){
 113                                         System.out.println("========================================================");
 114                                         logger.warn("There are taxa that have a 'is synonym of' - relationship but having taxon status 'accepted'!");
 115                                         System.out.println("========================================================");
 116                                 }
 117                                 int rIdentifier = rs.getInt("RIdentifier");
 118                                 int nameFk = rs.getInt("PTNameFk");
 119                                 int refFk = rs.getInt("PTRefFk");
 120                                 int relPTaxonId = rs.getInt("relPTaxonId");
 121                                 String taxonName = rs.getString("FullNameCache");
 122
 123                                 System.out.println("RIdentifier:" + rIdentifier + "\n  name: " + nameFk +
 124                                                 "\n  taxonName: " + taxonName + "\n  refId: " + refFk + "\n  RelPTaxonId: " + relPTaxonId );
 125                                 result = firstRow = false;
 126                         }
 127                         if (i > 0){
 128                                 System.out.println(" ");
 129                         }
 130
 131                         return result;
 132                 } catch (SQLException e) {
 133                         e.printStackTrace();
 134                         return false;
 135                 }
 136         }
 137
 138         private boolean checkInactivated(BerlinModelImportConfigurator bmiConfig){
 139                 try {
 140                         boolean result = true;
 141                         Source source = bmiConfig.getSource();
 142                         String strSQL = " SELECT * "  +
 143                                 " FROM PTaxon " +
 144                                         " INNER JOIN Name ON PTaxon.PTNameFk = Name.NameId " +
 145                                 " WHERE (PTaxon.DoubtfulFlag = 'i') ";
 146                         ResultSet rs = source.getResultSet(strSQL);
 147                         boolean firstRow = true;
 148                         int i = 0;
 149                         while (rs.next()){
 150                                 i++;
 151                                 if (firstRow){
 152                                         System.out.println("========================================================");
 153                                         logger.warn("There are taxa that have a doubtful flag 'i'(inactivated). Inactivated is not supported by CDM!");
 154                                         System.out.println("========================================================");
 155                                 }
 156                                 int rIdentifier = rs.getInt("RIdentifier");
 157                                 int nameFk = rs.getInt("PTNameFk");
 158                                 int refFk = rs.getInt("PTRefFk");
 159                                 String taxonName = rs.getString("FullNameCache");
 160
 161                                 System.out.println("RIdentifier:" + rIdentifier + "\n  nameId: " + nameFk +
 162                                                 "\n  taxonName: " + taxonName + "\n  refId: " + refFk  );
 163                                 result = firstRow = false;
 164                         }
 165                         if (i > 0){
 166                                 System.out.println(" ");
 167                         }
 168
 169                         return result;
 170                 } catch (SQLException e) {
 171                         e.printStackTrace();
 172                         return false;
 173                 }
 174         }
 175
 176         /* (non-Javadoc)
 177          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IImportConfigurator, eu.etaxonomy.cdm.api.application.CdmApplicationController, java.util.Map)
 178          */
 179         @Override
 180         protected boolean doInvoke(BerlinModelImportState state){
 181                 boolean success = true;
 182
 183                 //make not needed maps empty
 184                 String teamStore = ICdmIO.TEAM_STORE;
 185                 MapWrapper<? extends CdmBase> store = state.getStore(teamStore);
 186                 MapWrapper<TeamOrPersonBase> teamMap = (MapWrapper<TeamOrPersonBase>)store;
 187                 teamMap.makeEmpty();
 188
 189
 190                 MapWrapper<TaxonNameBase<?,?>> taxonNameMap = (MapWrapper<TaxonNameBase<?,?>>)state.getStore(ICdmIO.TAXONNAME_STORE);
 191                 MapWrapper<ReferenceBase> referenceMap = (MapWrapper<ReferenceBase>)state.getStore(ICdmIO.REFERENCE_STORE);
 192                 MapWrapper<ReferenceBase> nomRefMap = (MapWrapper<ReferenceBase>)state.getStore(ICdmIO.NOMREF_STORE);
 193                 MapWrapper<TaxonBase> taxonMap = (MapWrapper<TaxonBase>)state.getStore(ICdmIO.TAXON_STORE);
 194
 195                 BerlinModelImportConfigurator config = state.getConfig();
 196                 Source source = config.getSource();
 197
 198                 logger.info("start makeTaxa ...");
 199
 200                 String namespace = "PTaxon";
 201
 202                 try {
 203                         //get data from database
 204                         String strQuery =
 205                                         " SELECT * " +
 206                                         " FROM PTaxon " +
 207                                         " WHERE (1=1)";
 208
 209                         ResultSet rs = source.getResultSet(strQuery) ;
 210
 211                         int i = 0;
 212                         //for each reference
 213                         while (rs.next()){
 214
 215                                 if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("PTaxa handled: " + (i-1));}
 216
 217                                 //create TaxonName element
 218                                 int taxonId = rs.getInt("RIdentifier");
 219                                 int statusFk = rs.getInt("statusFk");
 220
 221                                 int nameFk = rs.getInt("PTNameFk");
 222                                 int refFk = rs.getInt("PTRefFk");
 223                                 String doubtful = rs.getString("DoubtfulFlag");
 224                                 String uuid = null;
 225                                 if (resultSetHasColumn(rs,"UUID")){
 226                                         uuid = rs.getString("UUID");
 227                                 }
 228
 229                                 TaxonNameBase<?,?> taxonName = null;
 230                                 if (taxonNameMap != null){
 231                                         taxonName  = taxonNameMap.get(nameFk);
 232                                 }
 233
 234                                 ReferenceBase<?> reference = null;
 235                                 if (referenceMap != null){
 236                                         reference = referenceMap.get(refFk);
 237                                         if (reference == null){
 238                                                 reference = nomRefMap.get(refFk);
 239                                         }
 240                                 }
 241
 242                                 if(! config.isIgnoreNull()){
 243                                         if (taxonName == null ){
 244                                                 logger.warn("TaxonName belonging to taxon (RIdentifier = " + taxonId + ") could not be found in store. Taxon will not be transported");
 245                                                 success = false;
 246                                                 continue; //next taxon
 247                                         }else if (reference == null ){
 248                                                 logger.warn("Reference belonging to taxon could not be found in store. Taxon will not be imported");
 249                                                 success = false;
 250                                                 continue; //next taxon
 251                                         }
 252                                 }
 253                                 TaxonBase<?> taxonBase;
 254                                 Synonym synonym;
 255                                 Taxon taxon;
 256                                 try {
 257                                         logger.debug(statusFk);
 258                                         if (statusFk == T_STATUS_ACCEPTED){
 259                                                 taxon = Taxon.NewInstance(taxonName, reference);
 260                                                 taxonBase = taxon;
 261                                         }else if (statusFk == T_STATUS_SYNONYM || statusFk == T_STATUS_PRO_PARTE_SYN || statusFk == T_STATUS_PARTIAL_SYN){
 262                                                 synonym = Synonym.NewInstance(taxonName, reference);
 263                                                 taxonBase = synonym;
 264                                                 if (statusFk == T_STATUS_PRO_PARTE_SYN){
 265                                                         config.addProParteSynonym(synonym);
 266                                                 }
 267                                                 if (statusFk == T_STATUS_PARTIAL_SYN){
 268                                                         config.addPartialSynonym(synonym);
 269                                                 }
 270                                         }else{
 271                                                 logger.warn("TaxonStatus " + statusFk + " not yet implemented. Taxon (RIdentifier = " + taxonId + ") left out.");
 272                                                 success = false;
 273                                                 continue;
 274                                         }
 275                                         if (uuid != null){
 276                                                 taxonBase.setUuid(UUID.fromString(uuid));
 277                                         }
 278
 279
 280                                         if (doubtful.equals("a")){
 281                                                 taxonBase.setDoubtful(false);
 282                                         }else if(doubtful.equals("d")){
 283                                                 taxonBase.setDoubtful(true);
 284                                         }else if(doubtful.equals("i")){
 285                                                 taxonBase.setDoubtful(false);
 286                                                 logger.warn("Doubtful = i (inactivated) does not exist in CDM. Doubtful set to false");
 287                                         }
 288
 289                                         //nameId
 290                                         ImportHelper.setOriginalSource(taxonBase, config.getSourceReference(), taxonId, namespace);
 291
 292                                         //detail
 293                                         String detail = rs.getString("Detail");
 294                                         if (CdmUtils.isNotEmpty(detail)){
 295                                                 ExtensionType detailExtensionType = getExtensionType(DETAIL_EXT_UUID, "micro reference","micro reference","micro ref.");
 296                                                 Extension.NewInstance(taxonBase, detail, detailExtensionType);
 297                                         }
 298                                         //idInSource
 299                                         String idInSource = rs.getString("IdInSource");
 300                                         if (CdmUtils.isNotEmpty(idInSource)){
 301                                                 ExtensionType detailExtensionType = getExtensionType(ID_IN_SOURCE_EXT_UUID, "Berlin Model IdInSource","Berlin Model IdInSource","BM source id");
 302                                                 Extension.NewInstance(taxonBase, idInSource, detailExtensionType);
 303                                         }
 304                                         //namePhrase
 305                                         String namePhrase = rs.getString("NamePhrase");
 306                                         if (CdmUtils.isNotEmpty(namePhrase)){
 307                                                 taxonBase.setAppendedPhrase(namePhrase);
 308                                         }
 309                                         //useNameCache
 310                                         Boolean useNameCacheFlag = rs.getBoolean("UseNameCacheFlag");
 311                                         if (useNameCacheFlag){
 312                                                 taxonBase.setUseNameCache(true);
 313                                         }
 314                                         //publisheFlag
 315                                         Boolean publishFlag = rs.getBoolean("PublishFlag");
 316                                         if (config.getTaxonPublishMarker().doMark(publishFlag)){
 317                                                 taxonBase.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), publishFlag));
 318                                         }
 319                                         //Notes
 320                                         doIdCreatedUpdatedNotes(state, taxonBase, rs, taxonId, namespace);
 321
 322                                         taxonMap.put(taxonId, taxonBase);
 323                                 } catch (Exception e) {
 324                                         logger.warn("An exception occurred when creating taxon with id " + taxonId + ". Taxon could not be saved.");
 325                                         success = false;
 326                                 }
 327                         }
 328                         //invokeRelations(source, cdmApp, deleteAll, taxonMap, referenceMap);
 329                         logger.info("saving "+i+" taxa ...");
 330                         getTaxonService().save(taxonMap.objects());
 331
 332                         logger.info("end makeTaxa ..." + getSuccessString(success));
 333
 334                         return success;
 335                 } catch (SQLException e) {
 336                         logger.error("SQLException:" +  e);
 337                         return false;
 338                 }
 339
 340         }
 341
 342
 343
 344         /* (non-Javadoc)
 345          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 346          */
 347         protected boolean isIgnore(BerlinModelImportState state){
 348                 return ! state.getConfig().isDoTaxa();
 349         }
 350
 351 }