fauna europaea import fix

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java
index 415e4a044be11904be98e51964e9975c7c9cec5f..18336d34a8246836e04b300a2794c39b5c330163 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java
@@ -43,4 +43,34 @@ public abstract class CdmImportBase<CONFIG extends IImportConfigurator, STATE ex
                return tree;\r
        }\r
        \r
+       \r
+       /**\r
+        * Alternative memory saving method variant of\r
+        * {@link #makeTree(STATE state, ReferenceBase ref)} which stores only the\r
+        * UUID instead of the full tree in the <code>ImportStateBase</code> by \r
+        * using <code>state.putTreeUuid(ref, tree);</code>\r
+        * \r
+        * @param state\r
+        * @param ref\r
+        * @return\r
+        */\r
+       protected TaxonomicTree makeTreeMemSave(STATE state, ReferenceBase ref){\r
+               String treeName = "TaxonTree (Import)";\r
+               if (ref != null && CdmUtils.isNotEmpty(ref.getTitleCache())){\r
+                       treeName = ref.getTitleCache();\r
+               }\r
+               TaxonomicTree tree = TaxonomicTree.NewInstance(treeName);\r
+               tree.setReference(ref);\r
+               \r
+\r
+               // use defined uuid for first tree\r
+               CONFIG config = (CONFIG)state.getConfig();\r
+               if (state.countTrees() < 1 ){\r
+                       tree.setUuid(config.getTreeUuid());\r
+               }\r
+               getTaxonService().saveTaxonomicTree(tree);\r
+               state.putTreeUuid(ref, tree);\r
+               return tree;\r
+       }\r
+       \r
 }\r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java
index aaf6a2a5697582029721aac06482bffd30def5af..555c0419fb36b21e24e80c1b11480d8a0d7eab06 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java
@@ -12,6 +12,7 @@ package eu.etaxonomy.cdm.io.common;
 \r
 import java.util.HashMap;\r
 import java.util.Map;\r
+import java.util.UUID;\r
 \r
 import org.apache.log4j.Logger;\r
 \r
@@ -37,6 +38,9 @@ public abstract class ImportStateBase<CONFIG extends ImportConfiguratorBase> ext
        \r
        private Map<ReferenceBase,TaxonomicTree> treeMap = new HashMap<ReferenceBase,TaxonomicTree>();\r
 \r
+       private Map<ReferenceBase,UUID> treeUuidMap = new HashMap<ReferenceBase,UUID>();\r
+\r
+       \r
        protected ImportStateBase(CONFIG config){\r
                this.config = config;\r
                stores.put(ICdmIO.USER_STORE, new MapWrapper<User>(service));\r
@@ -93,6 +97,23 @@ public abstract class ImportStateBase<CONFIG extends ImportConfiguratorBase> ext
        }\r
        \r
        public int countTrees(){\r
-               return treeMap.size();\r
+               return treeUuidMap.size();\r
+       }\r
+       \r
+       /**\r
+        * @return the treeUuid\r
+        */\r
+       public UUID getTreeUuid(ReferenceBase ref) {\r
+               return treeUuidMap.get(ref);\r
+       }\r
+\r
+       public void putTreeUuid(ReferenceBase ref, TaxonomicTree tree) {\r
+               if (tree != null){\r
+                       this.treeUuidMap.put(ref, tree.getUuid());\r
+               }\r
+       }\r
+       \r
+       public int countTreeUuids(){\r
+               return treeUuidMap.size();\r
        }\r
 }\r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
index c57c77b9b385d26689ffeca4f42c70d3849263c5..7267a995ae1f1fd4da54a566157730e7c7d83038 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
@@ -53,9 +53,9 @@ public class FaunaEuropaeaImportConfigurator extends ImportConfiguratorBase<Faun
        @SuppressWarnings("unchecked")\r
        protected void makeIoClassList() {\r
                ioClassList = new Class[] {\r
-                               FaunaEuropaeaAuthorImport.class,\r
-                               FaunaEuropaeaTaxonNameImport.class,\r
-//                             FaunaEuropaeaRelTaxonIncludeImport.class\r
+//                             FaunaEuropaeaAuthorImport.class,\r
+//                             FaunaEuropaeaTaxonNameImport.class,\r
+                               FaunaEuropaeaRelTaxonIncludeImport.class\r
 //                             FaunaEuropaeaRefImport.class,\r
 //                             FaunaEuropaeaDistributionImport.class\r
                };\r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRelTaxonIncludeImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRelTaxonIncludeImport.java
index 0ece881ef583502e7a460b3edfcb169b9634094a..49bb1bde5e07d3c553922f0eaa58d43a37c5aa84 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRelTaxonIncludeImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRelTaxonIncludeImport.java
@@ -152,7 +152,8 @@ public class FaunaEuropaeaRelTaxonIncludeImport extends FaunaEuropaeaImportBase
                                " SELECT dbo.Taxon.UUID AS ChildUuid, Parent.UUID AS ParentUuid " +\r
                                " FROM dbo.Taxon INNER JOIN dbo.Taxon AS Parent " +\r
                                " ON dbo.Taxon.TAX_TAX_IDPARENT = Parent.TAX_ID " +\r
-                               " WHERE (dbo.Taxon.TAX_VALID <> 0) AND (dbo.Taxon.TAX_AUT_ID <> " + A_AUCT + ")";\r
+                               " WHERE (dbo.Taxon.TAX_VALID <> 0) AND (dbo.Taxon.TAX_AUT_ID <> " + A_AUCT + ")" +\r
+                               " ORDER BY dbo.Taxon.TAX_RNK_ID ASC";\r
 \r
                        if (logger.isInfoEnabled()) {\r
                                logger.info("Query: " + strQuery);\r
@@ -162,9 +163,13 @@ public class FaunaEuropaeaRelTaxonIncludeImport extends FaunaEuropaeaImportBase
                        \r
                        while (rs.next()) {\r
                                \r
+                               // take memory snapshot every 10000 relations\r
+                               if((i % 10000) == 0){\r
+                                       ProfilerController.memorySnapshot();                                    \r
+                               }\r
+                               \r
                                if ((i++ % limit) == 0) { \r
                                        \r
-                                       ProfilerController.memorySnapshot();\r
                                        txStatus = startTransaction();\r
                                        childParentMap = new HashMap<UUID, UUID>(limit);\r
                                        \r
@@ -241,11 +246,14 @@ public class FaunaEuropaeaRelTaxonIncludeImport extends FaunaEuropaeaImportBase
                boolean success = true;\r
 \r
                        //add tree to new session\r
-                       TaxonomicTree tree = state.getTree(sourceRef);\r
-                       if (tree == null){\r
-                               tree = makeTree(state, sourceRef);\r
+               \r
+                       UUID treeUuid = state.getTreeUuid(sourceRef);\r
+                       TaxonomicTree tree;\r
+                       if (treeUuid == null){\r
+                               tree = makeTreeMemSave(state, sourceRef);\r
+                       } else {\r
+                               tree = getTaxonService().getTaxonomicTreeByUuid(treeUuid);                      \r
                        }\r
-                       getTaxonService().saveTaxonomicTree(tree);\r
                        \r
                        Set<TaxonBase> childSet = new HashSet<TaxonBase>(limit);\r
                        \r
@@ -257,7 +265,7 @@ public class FaunaEuropaeaRelTaxonIncludeImport extends FaunaEuropaeaImportBase
                        }\r
                        List<TaxonBase> children = getTaxonService().findByUuid(childKeysSet);\r
                        List<TaxonBase> parents = getTaxonService().findByUuid(parentValuesSet);\r
-                       Map<UUID, TaxonBase> parentsMap = new HashMap<UUID, TaxonBase>();\r
+                       Map<UUID, TaxonBase> parentsMap = new HashMap<UUID, TaxonBase>(parents.size());\r
                        for (TaxonBase taxonBase : parents){\r
                                parentsMap.put(taxonBase.getUuid(), taxonBase);\r
                        }\r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java
index e3c47d55205998a464c91f18b246f8eb956b8870..5482dccc850be34b36b7f378d73f45ede4a08141 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java
@@ -43,7 +43,6 @@ import eu.etaxonomy.cdm.io.common.IImportConfigurator;
 import eu.etaxonomy.cdm.io.common.ImportHelper;\r
 import eu.etaxonomy.cdm.io.common.MapWrapper;\r
 import eu.etaxonomy.cdm.io.common.Source;\r
-import eu.etaxonomy.cdm.io.profiler.ProfilerController;\r
 import eu.etaxonomy.cdm.io.tcsxml.in.TcsXmlImportState;\r
 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;\r
 import eu.etaxonomy.cdm.model.common.CdmBase;\r
@@ -138,6 +137,7 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
                \r
                success = retrieveTaxa(state, fauEuTaxonMap, Q_NO_RESTRICTION);\r
 //             success = processTaxa(state);\r
+//             ProfilerController.memorySnapshot();\r
                success = processTaxaFromDatabase(state, fauEuTaxonMap);\r
                \r
                commitTransaction(txStatus);\r
@@ -1392,8 +1392,6 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
 \r
                for (int j = 1; j <= n + 1; j++)\r
                {\r
-\r
-                       ProfilerController.memorySnapshot();\r
                        \r
                        int offset = j - 1;\r
                        int start = offset * limit;\r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java
index 158f96c3865dcab024a6b5d610ca98f8333e6be7..65bbc8d576353e2dc6cb87508b18ff0db6897242 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java
@@ -38,7 +38,6 @@ import eu.etaxonomy.cdm.io.common.ICdmIO;
 import eu.etaxonomy.cdm.io.common.ImportHelper;\r
 import eu.etaxonomy.cdm.io.common.MapWrapper;\r
 import eu.etaxonomy.cdm.io.common.Source;\r
-import eu.etaxonomy.cdm.io.profiler.ProfilerController;\r
 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;\r
 import eu.etaxonomy.cdm.model.common.CdmBase;\r
 import eu.etaxonomy.cdm.model.common.OriginalSource;\r
@@ -107,14 +106,12 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
         */\r
        protected boolean doInvoke(FaunaEuropaeaImportState state) {                            \r
                \r
-               ProfilerController.memorySnapshot();\r
                boolean success = true;\r
                if(logger.isInfoEnabled()) { logger.info("Start making taxa..."); }\r
 \r
                success = processTaxa(state);\r
                \r
                logger.info("End making taxa...");\r
-               ProfilerController.memorySnapshot();\r
                return success;\r
        }\r
 \r
@@ -157,7 +154,6 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
 \r
                                if ((i++ % limit) == 0) {\r
                                        \r
-                                       ProfilerController.memorySnapshot();\r
                                        txStatus = startTransaction();\r
                                        taxonMap = new HashMap<Integer, TaxonBase<?>>(limit);\r
                                        fauEuTaxonMap = new HashMap<Integer, FaunaEuropaeaTaxon>(limit);\r
@@ -297,8 +293,6 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase  {
 \r
                                        success = processTaxaSecondPass(state, taxonMap, fauEuTaxonMap);\r
                                        saveTaxa(state, taxonMap);\r
-\r
-                                       ProfilerController.memorySnapshot();\r
                                        \r
                                        taxonMap = null;\r
                                        fauEuTaxonMap = null;\r