partitioner for BM import

diff --git a/.gitattributes b/.gitattributes
index aed0390746267feb630cceb9a885bb41f3a598a6..e24a873a2c327f6c23b0734de11f163f959eb515 100644 (file)
--- a/.gitattributes
+++ b/.gitattributes
@@ -1205,6 +1205,8 @@ cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTypesImpor
 cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelUserImport.java -text
 cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelWebMarkerCategoryImport.java -text
 cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelWebMarkerImport.java -text
+cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/IPartitionedIO.java -text
+cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/ResultSetPartitioner.java -text
 cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/out/BerlinModelAuthorExport.java -text
 cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/out/BerlinModelAuthorTeamExport.java -text
 cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/out/BerlinModelExportBase.java -text

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/IPartitionedIO.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/IPartitionedIO.java
new file mode 100644 (file)
index 0000000..ccf0c74
--- /dev/null
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/IPartitionedIO.java
@@ -0,0 +1,40 @@
+// $Id$\r
+/**\r
+* Copyright (C) 2007 EDIT\r
+* European Distributed Institute of Taxonomy \r
+* http://www.e-taxonomy.eu\r
+* \r
+* The contents of this file are subject to the Mozilla Public License Version 1.1\r
+* See LICENSE.TXT at the top of this package for the full license terms.\r
+*/\r
+\r
+package eu.etaxonomy.cdm.io.berlinModel.in;\r
+\r
+import java.sql.ResultSet;\r
+import java.util.Map;\r
+\r
+import org.springframework.transaction.TransactionStatus;\r
+\r
+import eu.etaxonomy.cdm.model.common.CdmBase;\r
+\r
+\r
+/**\r
+ * @author a.mueller\r
+ * @created 16.02.2010\r
+ * @version 1.0\r
+ */\r
+public interface IPartitionedIO {\r
+       \r
+       //TODO make state more generic\r
+       public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state);\r
+       \r
+       public TransactionStatus startTransaction();\r
+       \r
+       public void commitTransaction(TransactionStatus txStatus);\r
+\r
+       /**\r
+        * @param rs\r
+        * @return\r
+        */\r
+       public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs);\r
+}\r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/ResultSetPartitioner.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/ResultSetPartitioner.java
new file mode 100644 (file)
index 0000000..56edd17
--- /dev/null
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/ResultSetPartitioner.java
@@ -0,0 +1,191 @@
+// $Id$\r
+/**\r
+* Copyright (C) 2007 EDIT\r
+* European Distributed Institute of Taxonomy \r
+* http://www.e-taxonomy.eu\r
+* \r
+* The contents of this file are subject to the Mozilla Public License Version 1.1\r
+* See LICENSE.TXT at the top of this package for the full license terms.\r
+*/\r
+\r
+package eu.etaxonomy.cdm.io.berlinModel.in;\r
+\r
+import java.sql.ResultSet;\r
+import java.sql.SQLException;\r
+import java.util.ArrayList;\r
+import java.util.HashMap;\r
+import java.util.List;\r
+import java.util.Map;\r
+\r
+import org.apache.log4j.Logger;\r
+import org.springframework.transaction.TransactionStatus;\r
+\r
+import eu.etaxonomy.cdm.common.CdmUtils;\r
+import eu.etaxonomy.cdm.io.common.Source;\r
+import eu.etaxonomy.cdm.model.common.CdmBase;\r
+\r
+/**\r
+ * @author a.mueller\r
+ * @created 16.02.2010\r
+ * @version 1.0\r
+ */\r
+public class ResultSetPartitioner {\r
+       private static final Logger logger = Logger.getLogger(ResultSetPartitioner.class);\r
+\r
+//************************* STATIC ***************************************************/\r
+       \r
+       public static ResultSetPartitioner NewInstance(Source source, String strIdQuery, String strRecordQuery, int partitionSize) throws SQLException{\r
+               ResultSetPartitioner resultSetPartitioner = new ResultSetPartitioner(source, strIdQuery, strRecordQuery, partitionSize);\r
+               return resultSetPartitioner;\r
+       }\r
+       \r
+//     public static ResultSetPartitioner NewInstance(ResultSet resultSet, int partitionSize) throws SQLException{\r
+//             ResultSetPartitioner resultSetPartitioner = new ResultSetPartitioner(resultSet, partitionSize);\r
+//             return resultSetPartitioner;\r
+//     }\r
+\r
+//*********************** VARIABLES *************************************************/\r
+       \r
+       private Source source;\r
+       \r
+       private ResultSet idResultSet;\r
+       \r
+       private String strRecordQueryTemplate;\r
+       \r
+       private ResultSet partitionResultSet;\r
+       \r
+       private Map<Object, Map<String, ? extends CdmBase>> relatedObjects;\r
+       \r
+       private int partitionSize;\r
+       \r
+       private List<Integer> currentIdList;\r
+       \r
+       private int currentPartition;\r
+       \r
+       private int currentRowInPartition;\r
+       \r
+       private TransactionStatus txStatus;\r
+       \r
+//*********************** CONSTRUCTOR *************************************************/\r
+\r
+       private ResultSetPartitioner(Source source, String strIdQuery, String strRecordQuery, int partitionSize) throws SQLException{\r
+               ResultSet idResultSet = source.getResultSet(strIdQuery);\r
+//             if (! idResultSet.isBeforeFirst()){\r
+//                     idResultSet.beforeFirst();\r
+//             }\r
+               this.source = source;\r
+               this.idResultSet = idResultSet;\r
+               this.strRecordQueryTemplate = strRecordQuery;\r
+               this.partitionSize = partitionSize;\r
+       }\r
+       \r
+//************************ METHODS ****************************************************/\r
+       \r
+\r
+       /**\r
+        * @param partitionedIO\r
+        */\r
+       public void doPartition(IPartitionedIO partitionedIO, BerlinModelImportState state) {\r
+               TransactionStatus txStatus = getTransaction(partitionSize, partitionedIO);\r
+               ResultSet rs = getPartitionResultSet();\r
+               this.relatedObjects = partitionedIO.getRelatedObjectsForPartition(rs);\r
+               partitionResultSet = getPartitionResultSet();\r
+               partitionedIO.doPartition(this, state);\r
+               partitionedIO.commitTransaction(txStatus);\r
+       }\r
+       \r
+       \r
+       public boolean nextPartition() throws SQLException{\r
+               currentPartition++;\r
+               if (idResultSet.isAfterLast()){\r
+                       return false;\r
+               }\r
+               currentIdList = new ArrayList<Integer>();\r
+               for (int i = 0; i < partitionSize; i++){\r
+                       if (idResultSet.next() == false){\r
+                               break; \r
+                       }\r
+                       int nextId = idResultSet.getInt(1);\r
+                       currentIdList.add(nextId);\r
+               }\r
+               currentRowInPartition = 0;\r
+               \r
+               return true;\r
+       }\r
+\r
+\r
+\r
+       /**\r
+        * Returns the underlying resultSet.<BR>\r
+        * @return\r
+        */\r
+       public ResultSet getResultSet(){\r
+               return partitionResultSet;\r
+       }\r
+\r
+       \r
+       \r
+       private ResultSet getPartitionResultSet(){\r
+               String strIdList = "";\r
+               for (Integer id: currentIdList){\r
+                       strIdList = CdmUtils.concat(",", strIdList, String.valueOf(id));\r
+               }\r
+               String strRecordQuery = strRecordQueryTemplate.replace("@IdList", strIdList);\r
+               ResultSet resultSet = source.getResultSet(strRecordQuery);\r
+               return resultSet;\r
+       }\r
+       \r
+       public Map<String, ? extends CdmBase> getObjectMap(Object key){\r
+               Map<String, ? extends CdmBase> objectMap = this.relatedObjects.get(key);\r
+               return objectMap;\r
+       }\r
+       \r
+\r
+       \r
+\r
+       /**\r
+        * @param recordsPerTransaction\r
+        * @param partitionedIO \r
+        * @param i\r
+        */\r
+       protected TransactionStatus getTransaction(int recordsPerTransaction, IPartitionedIO partitionedIO) {\r
+               //if (loopNeedsHandling (i, recordsPerTransaction) || txStatus == null) {\r
+                       txStatus = partitionedIO.startTransaction();\r
+                       if(logger.isInfoEnabled()) {\r
+                               logger.info("currentPartitionNumber = " + currentPartition + " - Transaction started"); \r
+                       }\r
+               //}\r
+               return txStatus;\r
+       }\r
+       \r
+// ************************** Not Needed ?? **************************************************\r
+       \r
+//     protected void doLogPerLoop(int recordsPerLog, String pluralString){\r
+//             int count = getAbsoluteRow() - 1;\r
+//             if ((count % recordsPerLog ) == 0 && count!= 0 ){ \r
+//                     logger.info(pluralString + " handled: " + (count));\r
+//             }\r
+//     }\r
+//\r
+//     \r
+//     /**\r
+//      * \r
+//      */\r
+//     private int getAbsoluteRow() {\r
+//             return partitionSize * (currentPartition - 1) + currentRowInPartition;\r
+//     }\r
+       \r
+\r
+//     public boolean nextRow() throws SQLException{\r
+//             if (currentRowInPartition >= partitionSize ){\r
+//                     return false;\r
+//             }else{\r
+//                     currentRowInPartition++;\r
+//                     return resultSet.next();\r
+//             }\r
+//     }\r
+//     \r
+       \r
+       \r
+       \r
+}\r