From: Andreas Müller Date: Wed, 17 Feb 2010 13:53:25 +0000 (+0000) Subject: partitioner for BM import X-Git-Tag: 3.0.3~1270 X-Git-Url: https://dev.e-taxonomy.eu/gitweb/cdmlib.git/commitdiff_plain/0e8018e09e44f359b0e08ebb327e8ec9e29e096e partitioner for BM import --- diff --git a/.gitattributes b/.gitattributes index aed0390746..e24a873a2c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1205,6 +1205,8 @@ cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTypesImpor cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelUserImport.java -text cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelWebMarkerCategoryImport.java -text cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelWebMarkerImport.java -text +cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/IPartitionedIO.java -text +cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/ResultSetPartitioner.java -text cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/out/BerlinModelAuthorExport.java -text cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/out/BerlinModelAuthorTeamExport.java -text cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/out/BerlinModelExportBase.java -text diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/IPartitionedIO.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/IPartitionedIO.java new file mode 100644 index 0000000000..ccf0c7412b --- /dev/null +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/IPartitionedIO.java @@ -0,0 +1,40 @@ +// $Id$ +/** +* Copyright (C) 2007 EDIT +* European Distributed Institute of Taxonomy +* http://www.e-taxonomy.eu +* +* The contents of this file are subject to the Mozilla Public License Version 1.1 +* See LICENSE.TXT at the top of this package for the full license terms. +*/ + +package eu.etaxonomy.cdm.io.berlinModel.in; + +import java.sql.ResultSet; +import java.util.Map; + +import org.springframework.transaction.TransactionStatus; + +import eu.etaxonomy.cdm.model.common.CdmBase; + + +/** + * @author a.mueller + * @created 16.02.2010 + * @version 1.0 + */ +public interface IPartitionedIO { + + //TODO make state more generic + public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state); + + public TransactionStatus startTransaction(); + + public void commitTransaction(TransactionStatus txStatus); + + /** + * @param rs + * @return + */ + public Map> getRelatedObjectsForPartition(ResultSet rs); +} diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/ResultSetPartitioner.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/ResultSetPartitioner.java new file mode 100644 index 0000000000..56edd17ab8 --- /dev/null +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/ResultSetPartitioner.java @@ -0,0 +1,191 @@ +// $Id$ +/** +* Copyright (C) 2007 EDIT +* European Distributed Institute of Taxonomy +* http://www.e-taxonomy.eu +* +* The contents of this file are subject to the Mozilla Public License Version 1.1 +* See LICENSE.TXT at the top of this package for the full license terms. +*/ + +package eu.etaxonomy.cdm.io.berlinModel.in; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.springframework.transaction.TransactionStatus; + +import eu.etaxonomy.cdm.common.CdmUtils; +import eu.etaxonomy.cdm.io.common.Source; +import eu.etaxonomy.cdm.model.common.CdmBase; + +/** + * @author a.mueller + * @created 16.02.2010 + * @version 1.0 + */ +public class ResultSetPartitioner { + private static final Logger logger = Logger.getLogger(ResultSetPartitioner.class); + +//************************* STATIC ***************************************************/ + + public static ResultSetPartitioner NewInstance(Source source, String strIdQuery, String strRecordQuery, int partitionSize) throws SQLException{ + ResultSetPartitioner resultSetPartitioner = new ResultSetPartitioner(source, strIdQuery, strRecordQuery, partitionSize); + return resultSetPartitioner; + } + +// public static ResultSetPartitioner NewInstance(ResultSet resultSet, int partitionSize) throws SQLException{ +// ResultSetPartitioner resultSetPartitioner = new ResultSetPartitioner(resultSet, partitionSize); +// return resultSetPartitioner; +// } + +//*********************** VARIABLES *************************************************/ + + private Source source; + + private ResultSet idResultSet; + + private String strRecordQueryTemplate; + + private ResultSet partitionResultSet; + + private Map> relatedObjects; + + private int partitionSize; + + private List currentIdList; + + private int currentPartition; + + private int currentRowInPartition; + + private TransactionStatus txStatus; + +//*********************** CONSTRUCTOR *************************************************/ + + private ResultSetPartitioner(Source source, String strIdQuery, String strRecordQuery, int partitionSize) throws SQLException{ + ResultSet idResultSet = source.getResultSet(strIdQuery); +// if (! idResultSet.isBeforeFirst()){ +// idResultSet.beforeFirst(); +// } + this.source = source; + this.idResultSet = idResultSet; + this.strRecordQueryTemplate = strRecordQuery; + this.partitionSize = partitionSize; + } + +//************************ METHODS ****************************************************/ + + + /** + * @param partitionedIO + */ + public void doPartition(IPartitionedIO partitionedIO, BerlinModelImportState state) { + TransactionStatus txStatus = getTransaction(partitionSize, partitionedIO); + ResultSet rs = getPartitionResultSet(); + this.relatedObjects = partitionedIO.getRelatedObjectsForPartition(rs); + partitionResultSet = getPartitionResultSet(); + partitionedIO.doPartition(this, state); + partitionedIO.commitTransaction(txStatus); + } + + + public boolean nextPartition() throws SQLException{ + currentPartition++; + if (idResultSet.isAfterLast()){ + return false; + } + currentIdList = new ArrayList(); + for (int i = 0; i < partitionSize; i++){ + if (idResultSet.next() == false){ + break; + } + int nextId = idResultSet.getInt(1); + currentIdList.add(nextId); + } + currentRowInPartition = 0; + + return true; + } + + + + /** + * Returns the underlying resultSet.
+ * @return + */ + public ResultSet getResultSet(){ + return partitionResultSet; + } + + + + private ResultSet getPartitionResultSet(){ + String strIdList = ""; + for (Integer id: currentIdList){ + strIdList = CdmUtils.concat(",", strIdList, String.valueOf(id)); + } + String strRecordQuery = strRecordQueryTemplate.replace("@IdList", strIdList); + ResultSet resultSet = source.getResultSet(strRecordQuery); + return resultSet; + } + + public Map getObjectMap(Object key){ + Map objectMap = this.relatedObjects.get(key); + return objectMap; + } + + + + + /** + * @param recordsPerTransaction + * @param partitionedIO + * @param i + */ + protected TransactionStatus getTransaction(int recordsPerTransaction, IPartitionedIO partitionedIO) { + //if (loopNeedsHandling (i, recordsPerTransaction) || txStatus == null) { + txStatus = partitionedIO.startTransaction(); + if(logger.isInfoEnabled()) { + logger.info("currentPartitionNumber = " + currentPartition + " - Transaction started"); + } + //} + return txStatus; + } + +// ************************** Not Needed ?? ************************************************** + +// protected void doLogPerLoop(int recordsPerLog, String pluralString){ +// int count = getAbsoluteRow() - 1; +// if ((count % recordsPerLog ) == 0 && count!= 0 ){ +// logger.info(pluralString + " handled: " + (count)); +// } +// } +// +// +// /** +// * +// */ +// private int getAbsoluteRow() { +// return partitionSize * (currentPartition - 1) + currentRowInPartition; +// } + + +// public boolean nextRow() throws SQLException{ +// if (currentRowInPartition >= partitionSize ){ +// return false; +// }else{ +// currentRowInPartition++; +// return resultSet.next(); +// } +// } +// + + + +}