Rework Fauna Europaea taxa import

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
index a79b4cba53546fe7b9db0f6ea457994dc3528e8b..e6e8fb14a7864404e64b890cb8f2d02f64f82244 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
@@ -53,10 +53,10 @@ public class FaunaEuropaeaImportConfigurator extends ImportConfiguratorBase<Faun
        @SuppressWarnings("unchecked")\r
        protected void makeIoClassList() {\r
                ioClassList = new Class[] {\r
-                               FaunaEuropaeaAuthorImport.class,\r
-                               FaunaEuropaeaNameImport.class,\r
+//                             FaunaEuropaeaAuthorImport.class,\r
+                               FaunaEuropaeaTaxonNameImport.class,\r
 //                             FaunaEuropaeaRelShipImport.class,\r
-                               FaunaEuropaeaRelTaxonIncludeImport.class\r
+//                             FaunaEuropaeaRelTaxonIncludeImport.class\r
 //                             FaunaEuropaeaRefImport.class,\r
 //                             FaunaEuropaeaDistributionImport.class\r
                };\r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java
index 7dfad14707ea17940cef70ee4162ab47796a7b62..cfdd549c350251434aed6d5d6c9a7bfa9b5af5af 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java
@@ -111,22 +111,22 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
                \r
                ProfilerController.memorySnapshot();\r
 \r
-               TransactionStatus txStatus = null;\r
+//             TransactionStatus txStatus = null;\r
                boolean success = true;\r
                \r
                if(logger.isInfoEnabled()) { logger.info("Start making taxa..."); }\r
 \r
-               if (state.getConfig().isUseTransactions()) {\r
-                       txStatus = startTransaction();\r
-               }\r
+//             if (state.getConfig().isUseTransactions()) {\r
+//                     txStatus = startTransaction();\r
+//             }\r
                \r
                success = retrieveTaxa(state);\r
 //             success = processTaxaSecondPass(state);\r
 //             success = saveTaxa(state, state.getHighestTaxonIndex(), state.getConfig().getLimitSave());\r
                \r
-               if (state.getConfig().isUseTransactions()) {\r
-                       commitTransaction(txStatus);\r
-               }\r
+//             if (state.getConfig().isUseTransactions()) {\r
+//                     commitTransaction(txStatus);\r
+//             }\r
                \r
                logger.info("End making taxa...");\r
                ProfilerController.memorySnapshot();\r
@@ -139,17 +139,24 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
 \r
                int limit = state.getConfig().getLimitSave();\r
 \r
+               TransactionStatus txStatus = null;\r
+\r
                Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores();\r
-//             MapWrapper<TaxonBase> nameStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXONNAME_STORE);\r
-               MapWrapper<TaxonBase> taxonStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXON_STORE);\r
                MapWrapper<TeamOrPersonBase> authorStore = (MapWrapper<TeamOrPersonBase>)stores.get(ICdmIO.TEAM_STORE);\r
+               \r
+//             MapWrapper<TaxonBase> nameStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXONNAME_STORE);\r
+//             MapWrapper<TaxonBase> taxonStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXON_STORE);\r
 //             Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap = state.getFauEuTaxonMap();\r
-               Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap = new HashMap<Integer, FaunaEuropaeaTaxon>(limit);\r
-               state.setFauEuTaxonMap(fauEuTaxonMap);\r
+               \r
+               Map<Integer, TaxonBase<?>> taxonMap = null;\r
+               Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap = null;\r
+\r
+//             Map<Integer, TaxonBase<?>> taxonMap = new HashMap<Integer, TaxonBase<?>>(limit);\r
+//             Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap = new HashMap<Integer, FaunaEuropaeaTaxon>(limit);\r
+               \r
+//             state.setFauEuTaxonMap(fauEuTaxonMap);\r
+//             state.setFauEuTaxonMap(fauEuTaxonMap);\r
 \r
-               if (logger.isInfoEnabled()) {\r
-                       logger.info("Initial size of taxon store is: " + taxonStore.size());\r
-               }\r
                FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();\r
                ReferenceBase<?> sourceRef = fauEuConfig.getSourceReference();\r
 \r
@@ -173,11 +180,16 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
 \r
                        while (rs.next()) {\r
 \r
-//                             if ((i++ % limit) == 0 && i != 1 ) { \r
-//                             if(logger.isInfoEnabled()) {\r
-//                             logger.info("Taxa retrieved: " + (i-1)); \r
-//                             }\r
-//                             }\r
+                               if ((i++ % limit) == 0) { \r
+                                       if (state.getConfig().isUseTransactions()) {\r
+                                               txStatus = startTransaction();\r
+                                       }\r
+                                       taxonMap = new HashMap<Integer, TaxonBase<?>>(limit);\r
+                                       fauEuTaxonMap = new HashMap<Integer, FaunaEuropaeaTaxon>(limit);\r
+                                       if(logger.isInfoEnabled()) {\r
+                                               logger.info("i = " + i + " - Transaction started"); \r
+                                       }\r
+                               }\r
 \r
                                int taxonId = rs.getInt("TAX_ID");\r
                                String localName = rs.getString("TAX_NAME");\r
@@ -282,14 +294,14 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
                                        ImportHelper.setOriginalSource(zooName, fauEuConfig.getSourceReference(), taxonId, "TaxonName");\r
 \r
 \r
-                                       if (!taxonStore.containsId(taxonId)) {\r
+                                       if (!taxonMap.containsKey(taxonId)) {\r
                                                if (taxonBase == null) {\r
                                                        if (logger.isDebugEnabled()) { \r
                                                                logger.debug("Taxon base is null. Taxon (" + taxonId + ") ignored.");\r
                                                        }\r
                                                        continue;\r
                                                }\r
-                                               taxonStore.put(taxonId, taxonBase);\r
+                                               taxonMap.put(taxonId, taxonBase);\r
                                                fauEuTaxonMap.put(taxonId, fauEuTaxon);\r
 //                                             if (logger.isDebugEnabled()) { \r
 //                                             logger.debug("Stored taxon base (" + taxonId + ") " + localName); \r
@@ -304,24 +316,24 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
                                        e.printStackTrace();\r
                                }\r
 \r
-                               if ((i++ % limit) == 0 && i != 1 ) { \r
-                                       if(logger.isInfoEnabled()) {\r
-                                               logger.info("Taxa retrieved: " + (i-1)); \r
-                                       }\r
-                                       success = processTaxaSecondPass(state);\r
-                                       success = saveTaxa(state);\r
+                               if ((i % limit) == 0 && i != 1 ) { \r
 \r
-                                       if (logger.isInfoEnabled()) {\r
-                                               logger.info("End size of taxon Store is: " + taxonStore.size());\r
-                                       }\r
+                                       success = processTaxaSecondPass(state, taxonMap, fauEuTaxonMap);\r
+                                       success = saveTaxa(state,  taxonMap);\r
 \r
 //                                     taxonStore.makeNewMap((IService)getTaxonService());\r
-                                       taxonStore.makeEmpty();\r
-                                       fauEuTaxonMap = new HashMap<Integer, FaunaEuropaeaTaxon>(limit);\r
-                                       state.setFauEuTaxonMap(fauEuTaxonMap);\r
-\r
-                                       if (logger.isInfoEnabled()) {\r
-                                               logger.info("Final size of taxon Store is: " + taxonStore.size());\r
+                                       \r
+                                       if (logger.isDebugEnabled()) {\r
+                                               logger.debug("Final size of taxon store is: " + taxonMap.size());\r
+                                       }\r
+                                       taxonMap = null;\r
+                                       fauEuTaxonMap = null;\r
+                                       \r
+                                       if (state.getConfig().isUseTransactions()) {\r
+                                               commitTransaction(txStatus);\r
+                                       }\r
+                                       if(logger.isInfoEnabled()) {\r
+                                               logger.info("i = " + i + " - Transaction committed"); \r
                                        }\r
                                }\r
 \r
@@ -338,21 +350,23 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
        /**\r
         * Processes taxa from complete taxon store\r
         */\r
-       private boolean processTaxaSecondPass(FaunaEuropaeaImportState state) {\r
+       private boolean processTaxaSecondPass(FaunaEuropaeaImportState state, Map<Integer, TaxonBase<?>> taxonMap,\r
+                       Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap) {\r
 \r
-               if(logger.isInfoEnabled()) { logger.info("Processing taxa second pass..."); }\r
+               if(logger.isDebugEnabled()) { logger.debug("Processing taxa second pass..."); }\r
+\r
+//             Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores();\r
+//             MapWrapper<TaxonBase> taxonStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXON_STORE);\r
+//             Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap = state.getFauEuTaxonMap();\r
 \r
-               Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores();\r
-               MapWrapper<TaxonBase> taxonStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXON_STORE);\r
                FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();\r
                ReferenceBase<?> sourceRef = fauEuConfig.getSourceReference();\r
-               Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap = state.getFauEuTaxonMap();\r
-\r
+               \r
                boolean success = true;\r
 \r
-               for (int id : taxonStore.keySet())\r
+               for (int id : taxonMap.keySet())\r
                {\r
-                       TaxonBase<?> taxonBase = taxonStore.get(id);\r
+                       TaxonBase<?> taxonBase = taxonMap.get(id);\r
                        if (logger.isDebugEnabled()) { logger.debug("Taxon # " + id); }\r
                        \r
                        TaxonNameBase<?,?> taxonName = taxonBase.getName();\r
@@ -723,25 +737,18 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
        }\r
 \r
        \r
-       protected boolean saveTaxa(FaunaEuropaeaImportState state) {\r
+       protected boolean saveTaxa(FaunaEuropaeaImportState state, Map<Integer, TaxonBase<?>> taxonMap) {\r
 \r
                boolean success = true;\r
 \r
-               Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores();\r
-               MapWrapper<TaxonBase> taxonStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXON_STORE);\r
-               TransactionStatus txStatus = null;\r
-\r
-               if(logger.isInfoEnabled()) { logger.info("Saving taxa ..."); }\r
+//             Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores();\r
+//             MapWrapper<TaxonBase> taxonStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXON_STORE);\r
+               \r
+               if(logger.isDebugEnabled()) { logger.debug("Saving taxa ..."); }\r
 \r
-               if (state.getConfig().isUseTransactions()) {\r
-                       txStatus = startTransaction();\r
-               }\r
 \r
-               getTaxonService().saveTaxonAll(taxonStore.objects());\r
+               getTaxonService().saveTaxonAll(taxonMap.values());\r
 \r
-               if (state.getConfig().isUseTransactions()) {\r
-                       commitTransaction(txStatus);\r
-               }\r
 \r
                return success;\r
        }\r