cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRefImport.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.faunaEuropaea;
  11
  12 import java.sql.ResultSet;
  13 import java.sql.SQLException;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22 import org.springframework.stereotype.Component;
  23 import org.springframework.transaction.TransactionStatus;
  24
  25 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  26 import eu.etaxonomy.cdm.io.common.ImportHelper;
  27 import eu.etaxonomy.cdm.io.common.Source;
  28 import eu.etaxonomy.cdm.model.agent.Team;
  29 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  30 import eu.etaxonomy.cdm.model.common.CdmBase;
  31 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  32 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
  33 import eu.etaxonomy.cdm.model.description.Feature;
  34 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  35 import eu.etaxonomy.cdm.model.description.TextData;
  36 import eu.etaxonomy.cdm.model.reference.Generic;
  37 import eu.etaxonomy.cdm.model.reference.ReferenceBase;
  38 import eu.etaxonomy.cdm.model.taxon.Synonym;
  39 import eu.etaxonomy.cdm.model.taxon.Taxon;
  40 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  41
  42
  43 /**
  44  * @author a.babadshanjan
  45  * @created 12.05.2009
  46  * @version 1.0
  47  */
  48 @Component
  49 public class FaunaEuropaeaRefImport extends FaunaEuropaeaImportBase {
  50         private static final Logger logger = Logger.getLogger(FaunaEuropaeaRefImport.class);
  51
  52         /* Interval for progress info message when retrieving taxa */
  53         private int modCount = 10000;
  54
  55         /* (non-Javadoc)
  56          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)
  57          */
  58         @Override
  59         protected boolean doCheck(FaunaEuropaeaImportState state) {
  60                 boolean result = true;
  61                 FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
  62                 logger.warn("Checking for References not yet fully implemented");
  63                 result &= checkReferenceStatus(fauEuConfig);
  64
  65                 return result;
  66         }
  67
  68         private boolean checkReferenceStatus(FaunaEuropaeaImportConfigurator fauEuConfig) {
  69                 boolean result = true;
  70 //              try {
  71                 Source source = fauEuConfig.getSource();
  72                 String sqlStr = "";
  73 //              ResultSet rs = source.getResultSet(sqlStr);
  74                 return result;
  75 //              } catch (SQLException e) {
  76 //              e.printStackTrace();
  77 //              return false;
  78 //              }
  79         }
  80
  81         /* (non-Javadoc)
  82          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IImportConfigurator, eu.etaxonomy.cdm.api.application.CdmApplicationController, java.util.Map)
  83          */
  84         @Override
  85         protected boolean doInvoke(FaunaEuropaeaImportState state) {
  86
  87                 TransactionStatus txStatus = null;
  88                 List<TaxonBase> taxonList = null;
  89                 Set<UUID> taxonUuids = null;
  90                 Set<ReferenceBase> references = null;
  91                 Set<TeamOrPersonBase> authors = null;
  92                 Map<UUID, FaunaEuropaeaReferenceTaxon> fauEuTaxonMap = null;
  93                 int limit = state.getConfig().getLimitSave();
  94
  95                 FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
  96                 Source source = fauEuConfig.getSource();
  97
  98                 String namespace = "Reference";
  99                 boolean success = true;
 100                 int i = 0;
 101
 102                 String selectCount =
 103                         " SELECT count(*) ";
 104
 105                 String selectColumns =
 106                         " SELECT Reference.*, TaxRefs.*, Taxon.UUID  ";
 107
 108                 String fromClause =
 109                         " FROM TaxRefs " +
 110                         " INNER JOIN Reference ON Reference.ref_id = TaxRefs.trf_ref_id " +
 111                         " INNER JOIN Taxon ON TaxRefs.trf_tax_id = Taxon.TAX_ID ";
 112
 113                 String orderClause =
 114                         " ORDER BY TaxRefs.trf_tax_id";
 115
 116                 String countQuery =
 117                         selectCount + fromClause;
 118
 119                 String selectQuery =
 120                         selectColumns + fromClause + orderClause;
 121
 122                 if(logger.isInfoEnabled()) { logger.info("Start making References..."); }
 123
 124                 try {
 125                         ResultSet rs = source.getResultSet(countQuery);
 126                         rs.next();
 127                         int count = rs.getInt(1);
 128
 129                         rs = source.getResultSet(selectQuery);
 130
 131                 if (logger.isInfoEnabled()) {
 132                                 logger.info("Number of rows: " + count);
 133                                 logger.info("Count Query: " + countQuery);
 134                                 logger.info("Select Query: " + selectQuery);
 135                         }
 136
 137                         while (rs.next()) {
 138
 139                                 if ((i++ % limit) == 0) {
 140
 141                                         txStatus = startTransaction();
 142                                         taxonUuids = new HashSet<UUID>(limit);
 143                                         references = new HashSet<ReferenceBase>(limit);
 144                                         authors = new HashSet<TeamOrPersonBase>(limit);
 145                                         fauEuTaxonMap = new HashMap<UUID, FaunaEuropaeaReferenceTaxon>(limit);
 146
 147                                         if(logger.isInfoEnabled()) {
 148                                                 logger.info("i = " + i + " - Reference import transaction started");
 149                                         }
 150                                 }
 151
 152                                 int taxonId = rs.getInt("trf_tax_id");
 153                                 int refId = rs.getInt("ref_id");
 154                                 String refAuthor = rs.getString("ref_author");
 155                                 String year = rs.getString("ref_year");
 156                                 String title = rs.getString("ref_title");
 157                                 String refSource = rs.getString("ref_source");
 158                                 String page = rs.getString("trf_page");
 159                                 UUID currentTaxonUuid = null;
 160                                 if (resultSetHasColumn(rs, "UUID")){
 161                                         currentTaxonUuid = UUID.fromString(rs.getString("UUID"));
 162                                 } else {
 163                                         logger.error("Taxon (" + taxonId + ") without UUID ignored");
 164                                         continue;
 165                                 }
 166
 167                                 FaunaEuropaeaReference fauEuReference = new FaunaEuropaeaReference();
 168                                 fauEuReference.setTaxonUuid(currentTaxonUuid);
 169                                 fauEuReference.setReferenceId(refId);
 170                                 fauEuReference.setReferenceAuthor(refAuthor);
 171                                 fauEuReference.setReferenceYear(year);
 172                                 fauEuReference.setReferenceTitle(title);
 173                                 fauEuReference.setReferenceSource(refSource);
 174                                 fauEuReference.setPage(page);
 175
 176                                 if (!taxonUuids.contains(currentTaxonUuid)) {
 177                                         taxonUuids.add(currentTaxonUuid);
 178                                         FaunaEuropaeaReferenceTaxon fauEuReferenceTaxon =
 179                                                 new FaunaEuropaeaReferenceTaxon(currentTaxonUuid);
 180                                         fauEuTaxonMap.put(currentTaxonUuid, fauEuReferenceTaxon);
 181                                 } else {
 182                                         if (logger.isTraceEnabled()) {
 183                                                 logger.trace("Taxon (" + currentTaxonUuid + ") already stored.");
 184                                                 continue;
 185                                         }
 186                                 }
 187
 188                                 ReferenceBase<?> reference = null;
 189                                 TeamOrPersonBase<Team> author = null;
 190
 191                                 reference = Generic.NewInstance();
 192                                 reference.setTitleCache(title);
 193                                 reference.setDatePublished(ImportHelper.getDatePublished(year));
 194                                 author = Team.NewInstance();
 195                                 author.setTitleCache(refAuthor);
 196
 197                                 ImportHelper.setOriginalSource(reference, fauEuConfig.getSourceReference(), refId, namespace);
 198                                 ImportHelper.setOriginalSource(author, fauEuConfig.getSourceReference(), refId, namespace);
 199
 200                                 // Store reference
 201
 202                                 if (!references.contains(refId)) {
 203                                         if (reference == null) {
 204                                                 logger.warn("Reference is null");
 205                                         }
 206                                         references.add(reference);
 207                                         if (logger.isDebugEnabled()) {
 208                                                 logger.debug("Stored reference (" + refId + ") " + refAuthor);
 209                                         }
 210                                 } else {
 211                                         if (logger.isDebugEnabled()) {
 212                                                 logger.debug("Not imported reference with duplicated ref_id (" + refId +
 213                                                                 ") " + refAuthor);
 214                                         }
 215                                         continue;
 216                                 }
 217
 218                                 // Store author
 219
 220                                 if (!authors.contains(refId)) {
 221                                         if (refAuthor == null) {
 222                                                 logger.warn("Reference author is null");
 223                                         }
 224                                         authors.add(author);
 225                                         if (logger.isDebugEnabled()) {
 226                                                 logger.debug("Stored author (" + refId + ") " + refAuthor);
 227                                         }
 228
 229                                         // TODO:
 230 //                                      List<TeamOrPersonBase<Team>> matches = getCommonService().findMatching(author, null);
 231 //                                      if (matches.size() == 0) {
 232 //                                      authorStore.put(refId, author);
 233 //                                      if (logger.isDebugEnabled()) {
 234 //                                      logger.debug("Stored author (" + refId + ") " + refAuthor);
 235 //                                      }
 236 //                                      } else {
 237 //                                      if (logger.isDebugEnabled()) {
 238 //                                      logger.debug("Matching authors found. Not stored author (" + refId + ") " + refAuthor);
 239 //                                      }
 240 //                                      }
 241                                 } else {
 242                                         if (logger.isDebugEnabled()) {
 243                                                 logger.debug("Not imported author with duplicated aut_id (" + refId +
 244                                                                 ") " + refAuthor);
 245                                         }
 246                                 }
 247
 248                                 fauEuTaxonMap.get(currentTaxonUuid).addReference(fauEuReference);
 249
 250                                 Taxon taxon = null;
 251                                 if (((i % limit) == 0 && i != 1 ) || i == count) {
 252
 253                                         try {
 254
 255                                                 taxonList = getTaxonService().findByUuid(taxonUuids);
 256
 257                                                 for (TaxonBase taxonBase : taxonList) {
 258
 259                                                         // Create descriptions
 260
 261                                                         if (taxonBase == null) {
 262                                                                 if (logger.isDebugEnabled()) {
 263                                                                         logger.debug("TaxonBase is null (" + currentTaxonUuid + ")");
 264                                                                 }
 265                                                                 continue;
 266                                                         }
 267                                                         boolean isSynonym = taxonBase.isInstanceOf(Synonym.class);
 268                                                         if (isSynonym) {
 269                                                                 Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
 270                                                                 Set<Taxon> acceptedTaxa = syn.getAcceptedTaxa();
 271                                                                 if (acceptedTaxa.size() > 0) {
 272                                                                         taxon = syn.getAcceptedTaxa().iterator().next();
 273                                                                 } else {
 274                                                                         if (logger.isDebugEnabled()) {
 275                                                                                 logger.debug("Synonym (" + taxonBase.getUuid() + ") does not have accepted taxa");
 276                                                                         }
 277                                                                 }
 278                                                         } else {
 279                                                                 taxon = CdmBase.deproxy(taxonBase, Taxon.class);
 280                                                         }
 281
 282                                                         if (taxon != null) {
 283                                                                 TaxonDescription taxonDescription = null;
 284                                                                 Set<TaxonDescription> descriptions = taxon.getDescriptions();
 285                                                                 if (descriptions.size() > 0) {
 286                                                                         taxonDescription = descriptions.iterator().next();
 287                                                                 } else {
 288                                                                         taxonDescription = TaxonDescription.NewInstance();
 289                                                                         taxon.addDescription(taxonDescription);
 290                                                                 }
 291
 292
 293                                                                 UUID taxonUuid = taxonBase.getUuid();
 294                                                                 FaunaEuropaeaReferenceTaxon fauEuHelperTaxon = fauEuTaxonMap.get(taxonUuid);
 295
 296                                                                 for (FaunaEuropaeaReference storedReference : fauEuHelperTaxon.getReferences()) {
 297
 298                                                                         TextData textData = TextData.NewInstance(Feature.CITATION());
 299
 300                                                                         ReferenceBase citation = storedReference.getCdmReference();
 301                                                                         String microCitation = storedReference.getPage();
 302                                                                         DescriptionElementSource originalSource = DescriptionElementSource.NewInstance(null, null, citation, microCitation, null, null);
 303                                                                         if (isSynonym){
 304                                                                                 Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);
 305                                                                                 originalSource.setNameUsedInSource(syn.getName());
 306                                                                         }
 307                                                                         textData.addSource(originalSource);
 308                                                                         taxonDescription.addElement(textData);
 309                                                                 }
 310                                                         }
 311                                                 }
 312                                                 if(logger.isInfoEnabled()) {
 313                                                         logger.info("i = " + i + " - Transaction committed");
 314                                                 }
 315
 316                                                 // save taxa, references, and authors
 317                                                 getTaxonService().saveTaxonAll(taxonList);
 318                                                 getReferenceService().saveReferenceAll(references);
 319 //                                              getAgentService().saveAgentAll(authors);
 320
 321                                                 taxonUuids = null;
 322                                                 references = null;
 323                                                 authors = null;
 324                                                 taxonList = null;
 325                                                 fauEuTaxonMap = null;
 326                                                 commitTransaction(txStatus);
 327
 328                                         } catch (Exception e) {
 329                                                 logger.warn("An exception occurred when creating reference with id " + refId +
 330                                                 ". Reference could not be saved.");
 331                                         }
 332                                 }
 333                         }
 334                 } catch (SQLException e) {
 335                         logger.error("SQLException:" +  e);
 336                         success = false;
 337                 }
 338                 if(logger.isInfoEnabled()) { logger.info("End making references ..."); }
 339
 340                 return success;
 341         }
 342
 343
 344         /* (non-Javadoc)
 345          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 346          */
 347         protected boolean isIgnore(FaunaEuropaeaImportState state){
 348                 return (state.getConfig().getDoReferences() == IImportConfigurator.DO_REFERENCES.NONE);
 349         }
 350
 351 }