From 14fe0ad6495115ce9e41e9526a432b614a6a4bfb Mon Sep 17 00:00:00 2001 From: Andreas Kohlbecker Date: Tue, 8 Sep 2009 09:28:23 +0000 Subject: [PATCH] fauna europaea import fix --- .../cdm/io/common/CdmImportBase.java | 30 +++++++++++++++++++ .../cdm/io/common/ImportStateBase.java | 23 +++++++++++++- .../FaunaEuropaeaImportConfigurator.java | 6 ++-- .../FaunaEuropaeaRelTaxonIncludeImport.java | 22 +++++++++----- .../FaunaEuropaeaTaxonImport.java | 4 +-- .../FaunaEuropaeaTaxonNameImport.java | 6 ---- 6 files changed, 71 insertions(+), 20 deletions(-) diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java index 415e4a044b..18336d34a8 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java @@ -43,4 +43,34 @@ public abstract class CdmImportBaseImportStateBase by + * using state.putTreeUuid(ref, tree); + * + * @param state + * @param ref + * @return + */ + protected TaxonomicTree makeTreeMemSave(STATE state, ReferenceBase ref){ + String treeName = "TaxonTree (Import)"; + if (ref != null && CdmUtils.isNotEmpty(ref.getTitleCache())){ + treeName = ref.getTitleCache(); + } + TaxonomicTree tree = TaxonomicTree.NewInstance(treeName); + tree.setReference(ref); + + + // use defined uuid for first tree + CONFIG config = (CONFIG)state.getConfig(); + if (state.countTrees() < 1 ){ + tree.setUuid(config.getTreeUuid()); + } + getTaxonService().saveTaxonomicTree(tree); + state.putTreeUuid(ref, tree); + return tree; + } + } diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java index aaf6a2a569..555c0419fb 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java @@ -12,6 +12,7 @@ package eu.etaxonomy.cdm.io.common; import java.util.HashMap; import java.util.Map; +import java.util.UUID; import org.apache.log4j.Logger; @@ -37,6 +38,9 @@ public abstract class ImportStateBase ext private Map treeMap = new HashMap(); + private Map treeUuidMap = new HashMap(); + + protected ImportStateBase(CONFIG config){ this.config = config; stores.put(ICdmIO.USER_STORE, new MapWrapper(service)); @@ -93,6 +97,23 @@ public abstract class ImportStateBase ext } public int countTrees(){ - return treeMap.size(); + return treeUuidMap.size(); + } + + /** + * @return the treeUuid + */ + public UUID getTreeUuid(ReferenceBase ref) { + return treeUuidMap.get(ref); + } + + public void putTreeUuid(ReferenceBase ref, TaxonomicTree tree) { + if (tree != null){ + this.treeUuidMap.put(ref, tree.getUuid()); + } + } + + public int countTreeUuids(){ + return treeUuidMap.size(); } } diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java index c57c77b9b3..7267a995ae 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java @@ -53,9 +53,9 @@ public class FaunaEuropaeaImportConfigurator extends ImportConfiguratorBase 0) AND (dbo.Taxon.TAX_AUT_ID <> " + A_AUCT + ")"; + " WHERE (dbo.Taxon.TAX_VALID <> 0) AND (dbo.Taxon.TAX_AUT_ID <> " + A_AUCT + ")" + + " ORDER BY dbo.Taxon.TAX_RNK_ID ASC"; if (logger.isInfoEnabled()) { logger.info("Query: " + strQuery); @@ -162,9 +163,13 @@ public class FaunaEuropaeaRelTaxonIncludeImport extends FaunaEuropaeaImportBase while (rs.next()) { + // take memory snapshot every 10000 relations + if((i % 10000) == 0){ + ProfilerController.memorySnapshot(); + } + if ((i++ % limit) == 0) { - ProfilerController.memorySnapshot(); txStatus = startTransaction(); childParentMap = new HashMap(limit); @@ -241,11 +246,14 @@ public class FaunaEuropaeaRelTaxonIncludeImport extends FaunaEuropaeaImportBase boolean success = true; //add tree to new session - TaxonomicTree tree = state.getTree(sourceRef); - if (tree == null){ - tree = makeTree(state, sourceRef); + + UUID treeUuid = state.getTreeUuid(sourceRef); + TaxonomicTree tree; + if (treeUuid == null){ + tree = makeTreeMemSave(state, sourceRef); + } else { + tree = getTaxonService().getTaxonomicTreeByUuid(treeUuid); } - getTaxonService().saveTaxonomicTree(tree); Set childSet = new HashSet(limit); @@ -257,7 +265,7 @@ public class FaunaEuropaeaRelTaxonIncludeImport extends FaunaEuropaeaImportBase } List children = getTaxonService().findByUuid(childKeysSet); List parents = getTaxonService().findByUuid(parentValuesSet); - Map parentsMap = new HashMap(); + Map parentsMap = new HashMap(parents.size()); for (TaxonBase taxonBase : parents){ parentsMap.put(taxonBase.getUuid(), taxonBase); } diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java index e3c47d5520..5482dccc85 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java @@ -43,7 +43,6 @@ import eu.etaxonomy.cdm.io.common.IImportConfigurator; import eu.etaxonomy.cdm.io.common.ImportHelper; import eu.etaxonomy.cdm.io.common.MapWrapper; import eu.etaxonomy.cdm.io.common.Source; -import eu.etaxonomy.cdm.io.profiler.ProfilerController; import eu.etaxonomy.cdm.io.tcsxml.in.TcsXmlImportState; import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase; import eu.etaxonomy.cdm.model.common.CdmBase; @@ -138,6 +137,7 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase { success = retrieveTaxa(state, fauEuTaxonMap, Q_NO_RESTRICTION); // success = processTaxa(state); +// ProfilerController.memorySnapshot(); success = processTaxaFromDatabase(state, fauEuTaxonMap); commitTransaction(txStatus); @@ -1392,8 +1392,6 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase { for (int j = 1; j <= n + 1; j++) { - - ProfilerController.memorySnapshot(); int offset = j - 1; int start = offset * limit; diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java index 158f96c386..65bbc8d576 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonNameImport.java @@ -38,7 +38,6 @@ import eu.etaxonomy.cdm.io.common.ICdmIO; import eu.etaxonomy.cdm.io.common.ImportHelper; import eu.etaxonomy.cdm.io.common.MapWrapper; import eu.etaxonomy.cdm.io.common.Source; -import eu.etaxonomy.cdm.io.profiler.ProfilerController; import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase; import eu.etaxonomy.cdm.model.common.CdmBase; import eu.etaxonomy.cdm.model.common.OriginalSource; @@ -107,14 +106,12 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase { */ protected boolean doInvoke(FaunaEuropaeaImportState state) { - ProfilerController.memorySnapshot(); boolean success = true; if(logger.isInfoEnabled()) { logger.info("Start making taxa..."); } success = processTaxa(state); logger.info("End making taxa..."); - ProfilerController.memorySnapshot(); return success; } @@ -157,7 +154,6 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase { if ((i++ % limit) == 0) { - ProfilerController.memorySnapshot(); txStatus = startTransaction(); taxonMap = new HashMap>(limit); fauEuTaxonMap = new HashMap(limit); @@ -297,8 +293,6 @@ public class FaunaEuropaeaTaxonNameImport extends FaunaEuropaeaImportBase { success = processTaxaSecondPass(state, taxonMap, fauEuTaxonMap); saveTaxa(state, taxonMap); - - ProfilerController.memorySnapshot(); taxonMap = null; fauEuTaxonMap = null; -- 2.34.1