Fauna Europaea reference import improvements

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRefImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRefImport.java
index db55db3782a30d68d675066f799fe6fa31c459b8..2f87d46159d354cb601d58dada0558091921a76d 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRefImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaRefImport.java
@@ -14,6 +14,7 @@ import static eu.etaxonomy.cdm.io.faunaEuropaea.FaunaEuropaeaTransformer.T_STATU
 \r
 import java.sql.ResultSet;\r
 import java.sql.SQLException;\r
+import java.util.Arrays;\r
 import java.util.HashMap;\r
 import java.util.List;\r
 import java.util.Map;\r
@@ -21,6 +22,8 @@ import java.util.Set;
 import java.util.UUID;\r
 \r
 import org.apache.log4j.Logger;\r
+import org.hibernate.Hibernate;\r
+import org.springframework.beans.factory.annotation.Autowired;\r
 import org.springframework.stereotype.Component;\r
 import org.springframework.transaction.TransactionStatus;\r
 \r
@@ -51,6 +54,7 @@ import eu.etaxonomy.cdm.model.reference.StrictReferenceBase;
 import eu.etaxonomy.cdm.model.taxon.Synonym;\r
 import eu.etaxonomy.cdm.model.taxon.Taxon;\r
 import eu.etaxonomy.cdm.model.taxon.TaxonBase;\r
+import eu.etaxonomy.cdm.persistence.dao.BeanInitializer;\r
 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;\r
 import eu.etaxonomy.cdm.strategy.match.IMatchable;\r
 \r
@@ -64,9 +68,19 @@ import eu.etaxonomy.cdm.strategy.match.IMatchable;
 public class FaunaEuropaeaRefImport extends FaunaEuropaeaImportBase {\r
        private static final Logger logger = Logger.getLogger(FaunaEuropaeaRefImport.class);\r
 \r
+//     private static final List<String> TAXON_INIT_STRATEGY = Arrays.asList(new String []{\r
+//                     "descriptions.*"\r
+//                     });\r
+//\r
+//     private static final List<String> SYNONYM_INIT_STRATEGY = Arrays.asList(new String []{\r
+//                     "synonymRelations"\r
+//                     });\r
+       \r
        /* Interval for progress info message when retrieving taxa */\r
        private int modCount = 10000;\r
        \r
+       @Autowired\r
+       private BeanInitializer defaultBeanInitializer;\r
                \r
        /* (non-Javadoc)\r
         * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)\r
@@ -102,7 +116,7 @@ public class FaunaEuropaeaRefImport extends FaunaEuropaeaImportBase {
                \r
                Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores();\r
                MapWrapper<TeamOrPersonBase> authorStore = (MapWrapper<TeamOrPersonBase>)stores.get(ICdmIO.TEAM_STORE);\r
-               MapWrapper<TaxonBase> taxonStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXON_STORE);\r
+//             MapWrapper<TaxonBase> taxonStore = (MapWrapper<TaxonBase>)stores.get(ICdmIO.TAXON_STORE);\r
                MapWrapper<ReferenceBase> refStore = (MapWrapper<ReferenceBase>)stores.get(ICdmIO.REFERENCE_STORE);\r
                TransactionStatus txStatus = null;\r
                                \r
@@ -116,11 +130,15 @@ public class FaunaEuropaeaRefImport extends FaunaEuropaeaImportBase {
                \r
                try {\r
                        String strQuery = \r
-                               " SELECT Reference.*, TaxRefs.* " + \r
-                " FROM Reference INNER JOIN TaxRefs ON Reference.ref_id = TaxRefs.trf_ref_id " +\r
-                " WHERE (1=1)" + \r
+                               " SELECT Reference.*, TaxRefs.*, Taxon.UUID " + \r
+                " FROM TaxRefs " +\r
+                               " INNER JOIN Reference ON Reference.ref_id = TaxRefs.trf_ref_id " +\r
+                " INNER JOIN Taxon ON TaxRefs.trf_tax_id = Taxon.TAX_ID " +\r
                 " ORDER BY TaxRefs.trf_tax_id";\r
                        \r
+                       if (logger.isInfoEnabled()) {\r
+                               logger.info("Query: " + strQuery);\r
+                       }\r
                        ResultSet rs = source.getResultSet(strQuery) ;\r
                        \r
                        int i = 0;\r
@@ -139,7 +157,15 @@ public class FaunaEuropaeaRefImport extends FaunaEuropaeaImportBase {
                                String title = rs.getString("ref_title");\r
                                String refSource = rs.getString("ref_source");\r
                                String page = rs.getString("trf_page");\r
+                               UUID taxonBaseUuid = null;\r
+                               if (resultSetHasColumn(rs,"UUID")){\r
+                                       taxonBaseUuid = UUID.fromString(rs.getString("UUID"));\r
+                               } else {\r
+                                       taxonBaseUuid = UUID.randomUUID();\r
+                               }\r
                                \r
+                               txStatus = startTransaction();\r
+\r
                                StrictReferenceBase<?> reference = null;\r
                                TeamOrPersonBase<Team> author = null;\r
                                \r
@@ -173,16 +199,21 @@ public class FaunaEuropaeaRefImport extends FaunaEuropaeaImportBase {
                                        \r
                                        // Create authors\r
                                        \r
-                                       if (!authorStore.containsId(refId)) { // TODO: Don't insert identical author names\r
+                                       if (!authorStore.containsId(refId)) {\r
                                                if (refAuthor == null) {\r
                                                        logger.warn("Reference author is null");\r
                                                }\r
                                                \r
                                                List<TeamOrPersonBase<Team>> matches = getCommonService().findMatching(author, null);\r
-                                                       \r
-                                               authorStore.put(refId, author);\r
-                                               if (logger.isDebugEnabled()) { \r
-                                                       logger.debug("Stored author (" + refId + ") " + refAuthor); \r
+                                               if (matches.size() == 0) {\r
+                                                       authorStore.put(refId, author);\r
+                                                       if (logger.isDebugEnabled()) { \r
+                                                               logger.debug("Stored author (" + refId + ") " + refAuthor); \r
+                                                       }\r
+                                               } else {\r
+                                                       if (logger.isDebugEnabled()) { \r
+                                                               logger.debug("Matching authors found. Not stored author (" + refId + ") " + refAuthor); \r
+                                                       }\r
                                                }\r
                                        } else {\r
                                                if (logger.isDebugEnabled()) { \r
@@ -199,34 +230,53 @@ public class FaunaEuropaeaRefImport extends FaunaEuropaeaImportBase {
                                try {\r
                                        // Create descriptions\r
 \r
-                                       Taxon taxon;\r
-                                       TaxonBase taxonBase = taxonStore.get(taxonId);\r
-                                       if (taxonBase == null) { continue; }\r
+                                       Taxon taxon = null;\r
+                                       TaxonBase taxonBase = getTaxonService().findByUuid(taxonBaseUuid);\r
+//                                     TaxonBase taxonBase = getTaxonService().load(taxonBaseUuid, TAXON_INIT_STRATEGY);\r
+//                                     TaxonBase taxonBase = taxonStore.get(taxonId);\r
+                                       if (taxonBase == null) { \r
+                                               if (logger.isDebugEnabled()) { \r
+                                                       logger.debug("TaxonBase is null (" + taxonBaseUuid);\r
+                                               }\r
+                                               continue; \r
+                                       }\r
                                        boolean isSynonym = taxonBase.isInstanceOf(Synonym.class);\r
-                                       if (isSynonym){\r
+                                       if (isSynonym) {\r
+//                                             taxonBase = getTaxonService().load(taxonBase.getUuid(), SYNONYM_INIT_STRATEGY);\r
                                                Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);\r
-                                               taxon = syn.getAcceptedTaxa().iterator().next();\r
-                                       }else{\r
+                                               Set<Taxon> acceptedTaxa = syn.getAcceptedTaxa();\r
+                                               if (acceptedTaxa.size() > 0) {\r
+                                                       taxon = syn.getAcceptedTaxa().iterator().next();\r
+                                               } else {\r
+                                                       if (logger.isDebugEnabled()) { \r
+                                                               logger.debug("Synonym (" + taxonBase.getUuid() + ") does not have accepted taxa");\r
+                                                       }\r
+                                               }\r
+                                       } else {\r
                                                taxon = CdmBase.deproxy(taxonBase, Taxon.class);\r
                                        }\r
 \r
-                                       TaxonDescription taxonDescription = null;\r
-                                       Set<TaxonDescription> descriptions = taxon.getDescriptions();\r
-                                       if (descriptions.size() > 0) {\r
-                                               taxonDescription = descriptions.iterator().next(); \r
-                                       } else {\r
-                                               taxonDescription = TaxonDescription.NewInstance();\r
-                                               taxon.addDescription(taxonDescription);\r
-                                       }\r
+                                       if (taxon != null) {\r
+                                               TaxonDescription taxonDescription = null;\r
+//                                             Hibernate.initialize(taxon);\r
+//                                             defaultBeanInitializer.initialize(taxon, TAXON_INIT_DESCRIPTIONS_STRATEGY);\r
+                                               Set<TaxonDescription> descriptions = taxon.getDescriptions();\r
+                                               if (descriptions.size() > 0) {\r
+                                                       taxonDescription = descriptions.iterator().next(); \r
+                                               } else {\r
+                                                       taxonDescription = TaxonDescription.NewInstance();\r
+                                                       taxon.addDescription(taxonDescription);\r
+                                               }\r
 \r
-                                       TextData textData = TextData.NewInstance(Feature.CITATION());\r
-                                       if (isSynonym){\r
-                                               Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);\r
-                                               textData.setNameUsedInReference(syn.getName());\r
+                                               TextData textData = TextData.NewInstance(Feature.CITATION());\r
+                                               if (isSynonym){\r
+                                                       Synonym syn = CdmBase.deproxy(taxonBase, Synonym.class);\r
+                                                       textData.setNameUsedInReference(syn.getName());\r
+                                               }\r
+                                               textData.setCitation(reference);\r
+                                               textData.setCitationMicroReference(page);\r
+                                               taxonDescription.addElement(textData);\r
                                        }\r
-                                       textData.setCitation(reference);\r
-                                       textData.setCitationMicroReference(page);\r
-                                       taxonDescription.addElement(textData);\r
 \r
                                } catch (Exception e) {\r
                                        logger.warn("An exception occurred when creating description for reference " + refId + \r
@@ -238,8 +288,6 @@ public class FaunaEuropaeaRefImport extends FaunaEuropaeaImportBase {
                        \r
                        if(logger.isInfoEnabled()) { logger.info("Saving references ..."); }\r
                        \r
-                       txStatus = startTransaction();\r
-\r
                        // save taxa, references, and authors\r
                        success = saveTaxa(state, state.getHighestTaxonIndex(), state.getConfig().getLimitSave());\r
                        getReferenceService().saveReferenceAll(refStore.objects());\r