Fauna Europaea Import - 230.187 taxa imported

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
index ce59f6432ef70bc509486410f7364e74cc84d22f..0e6cc14bd0f5d9f036e5447aeca49a1c51b4639a 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaImportConfigurator.java
@@ -30,8 +30,8 @@ public class FaunaEuropaeaImportConfigurator extends ImportConfiguratorBase impl
        @SuppressWarnings("unchecked")\r
        protected void makeIoClassList() {\r
                ioClassList = new Class[] {\r
-//                             FaunaEuropaeaAuthorImport.class,\r
-//                             FaunaEuropaeaReferenceImport.class,\r
+                               FaunaEuropaeaAuthorImport.class,\r
+                               FaunaEuropaeaReferenceImport.class,\r
 //                             FaunaEuropaeaTaxonNameImport.class\r
                                FaunaEuropaeaTaxonImport.class\r
 //                             FaunaEuropaeaDistributionImport.class\r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaReferenceImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaReferenceImport.java
index 239dc5b1cef137a432b4fc22f7a1ebf29a2664e1..e4e7df0bfadfa130afe136546017d6ee1655d668 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaReferenceImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaReferenceImport.java
@@ -31,6 +31,7 @@ import eu.etaxonomy.cdm.io.common.IImportConfigurator;
 import eu.etaxonomy.cdm.io.common.ImportHelper;\r
 import eu.etaxonomy.cdm.io.common.MapWrapper;\r
 import eu.etaxonomy.cdm.io.common.Source;\r
+import eu.etaxonomy.cdm.model.agent.Team;\r
 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;\r
 import eu.etaxonomy.cdm.model.common.CdmBase;\r
 import eu.etaxonomy.cdm.model.name.Rank;\r
@@ -60,8 +61,6 @@ public class FaunaEuropaeaReferenceImport extends FaunaEuropaeaImportBase {
        /* Max number of references to be saved with one service call */\r
        private int limit = 20000; // TODO: Make configurable\r
        \r
-       public FaunaEuropaeaReferenceImport() {\r
-       }\r
                \r
        /* (non-Javadoc)\r
         * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)\r
@@ -132,19 +131,20 @@ public class FaunaEuropaeaReferenceImport extends FaunaEuropaeaImportBase {
                                }\r
                                \r
                                int refId = rs.getInt("ref_id");\r
-                               //String author = rs.getString("ref_author");\r
+                               String refAuthor = rs.getString("ref_author");\r
                                String year = rs.getString("ref_year");\r
                                String title = rs.getString("ref_title");\r
-                               String ref_author = rs.getString("ref_author");\r
                                String refSource = rs.getString("ref_source");\r
-//                             int authorId = rs.getInt("aut_id");\r
                                \r
                                StrictReferenceBase<?> reference = null;\r
+                               TeamOrPersonBase<Team> author = null;\r
                                \r
                                try {\r
                                        reference = Generic.NewInstance();\r
                                        reference.setTitleCache(title);\r
                                        reference.setDatePublished(ImportHelper.getDatePublished(year));\r
+                                       author = Team.NewInstance();\r
+                                       author.setTitleCache(refAuthor);\r
                                        \r
                                        // FIXME: author.aut_name and Reference.ref_author don't match\r
 //                                     if (authorStore != null) {\r
@@ -155,6 +155,9 @@ public class FaunaEuropaeaReferenceImport extends FaunaEuropaeaImportBase {
 //                                     }\r
                                                                                \r
                                        ImportHelper.setOriginalSource(reference, fauEuConfig.getSourceReference(), refId, namespace);\r
+                                       ImportHelper.setOriginalSource(author, fauEuConfig.getSourceReference(), refId, namespace);\r
+                                       \r
+                                       // Create reference\r
                                        \r
                                        if (!refStore.containsId(refId)) {\r
                                                if (reference == null) {\r
@@ -162,13 +165,29 @@ public class FaunaEuropaeaReferenceImport extends FaunaEuropaeaImportBase {
                                                }\r
                                                refStore.put(refId, reference);\r
                                                if (logger.isDebugEnabled()) { \r
-                                                       logger.debug("Stored reference (" + refId + ") " + ref_author); \r
+                                                       logger.debug("Stored reference (" + refId + ") " + refAuthor); \r
                                                }\r
                                        } else {\r
                                                logger.warn("Not imported reference with duplicated ref_id (" + refId + \r
-                                                               ") " + ref_author);\r
+                                                               ") " + refAuthor);\r
+                                       }\r
+                                       \r
+                                       // Create authors\r
+                                       \r
+                                       if (!authorStore.containsId(refId)) {\r
+                                               if (refAuthor == null) {\r
+                                                       logger.warn("Reference author is null");\r
+                                               }\r
+                                               authorStore.put(refId, author);\r
+                                               if (logger.isDebugEnabled()) { \r
+                                                       logger.debug("Stored author (" + refId + ") " + refAuthor); \r
+                                               }\r
+                                       } else {\r
+                                               logger.warn("Not imported author with duplicated aut_id (" + refId + \r
+                                                               ") " + refAuthor);\r
                                        }\r
                                        \r
+                                       \r
                                } catch (Exception e) {\r
                                        logger.warn("An exception occurred when creating reference with id " + refId + \r
                                        ". Reference could not be saved.");\r
@@ -177,8 +196,9 @@ public class FaunaEuropaeaReferenceImport extends FaunaEuropaeaImportBase {
                        \r
                        if(logger.isInfoEnabled()) { logger.info("Saving references ..."); }\r
                        \r
-                       // save references\r
+                       // save authors and references\r
                        getReferenceService().saveReferenceAll(refStore.objects());\r
+                       getAgentService().saveAgentAll(authorStore.objects());\r
                        \r
                        if(logger.isInfoEnabled()) { logger.info("End making references ..."); }\r
                        \r

diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java
index e35c13884e733b850d879e063124e0fa6256c30b..d719ac7b1b35a64bd4596e411375d695d9958719 100644 (file)
--- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java
+++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/faunaEuropaea/FaunaEuropaeaTaxonImport.java
@@ -64,7 +64,7 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
        private static final Logger logger = Logger.getLogger(FaunaEuropaeaTaxonImport.class);\r
 \r
        /* Max number of taxa to retrieve (for test purposes) */\r
-       private int maxTaxa = 5000;\r
+       private int maxTaxa = 0;\r
        /* Max number of taxa to be saved with one service call */\r
        private int limit = 10000; // TODO: Make configurable\r
        /* Interval for progress info message when retrieving taxa */\r
@@ -172,8 +172,8 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
                                " WHERE (1=1)" +\r
                                validClause;\r
 \r
-                       if (logger.isInfoEnabled()) {\r
-                               logger.info("Query: " + strQuery);\r
+                       if (logger.isDebugEnabled()) {\r
+                               logger.debug("Query: " + strQuery);\r
                        }\r
                        rs = source.getResultSet(strQuery);\r
                        \r
@@ -298,10 +298,10 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
                        TaxonBase<?> taxonBase = taxonStore.get(id);\r
                        FaunaEuropaeaTaxon fauEuTaxon = fauEuTaxonMap.get(id);\r
                        \r
-                       if (logger.isInfoEnabled()) { logger.info("Taxon # " + id); }\r
+                       if (logger.isDebugEnabled()) { logger.debug("Taxon # " + id); }\r
                        String nameString = calculateTaxonName(fauEuTaxon, taxonBase, taxonStore, fauEuTaxonMap);\r
 //                     String nameString = calculateTaxonName(fauEuTaxon, taxonBase, taxonStore);\r
-                       setTaxonName(nameString, fauEuTaxon, taxonBase);\r
+                       setTaxonName(nameString, fauEuTaxon, taxonBase, fauEuConfig);\r
                        \r
                }\r
                return success; \r
@@ -317,34 +317,6 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
                        subString = nameString.substring(0, index);\r
                }\r
                return subString;\r
-               \r
-//             String[] tokens = nameString.split("[\\s]+");\r
-//             StringBuilder stringBuilder = \r
-//             int len = tokens.length - 1;\r
-//             for (int i = 0; i < len - 1; i++) {\r
-//              String lastToken = tokens[len];\r
-//             }\r
-//             return lastToken;\r
-       }\r
-       \r
-       \r
-       /* Remove last part of name */\r
-       private String exchangeEpithet(String nameString) {\r
-               \r
-               String subString = nameString;\r
-               int index = nameString.lastIndexOf(" ");\r
-               if (index > 0) {\r
-                       subString = nameString.substring(0, index);\r
-               }\r
-               return subString;\r
-               \r
-//             String[] tokens = nameString.split("[\\s]+");\r
-//             StringBuilder stringBuilder = \r
-//             int len = tokens.length - 1;\r
-//             for (int i = 0; i < len - 1; i++) {\r
-//              String lastToken = tokens[len];\r
-//             }\r
-//             return lastToken;\r
        }\r
        \r
        \r
@@ -476,8 +448,8 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
                                parentConcatStringBuilder.append(concatString);\r
                                parentConcatString = parentConcatStringBuilder.toString();\r
 \r
-                               if (logger.isInfoEnabled()) { \r
-                                       logger.info("Concatenated name: " + parentConcatString); \r
+                               if (logger.isDebugEnabled()) { \r
+                                       logger.debug("Concatenated name: " + parentConcatString); \r
                                }\r
                        } else {\r
                                logger.warn("Parent uuid of " + parentId + " is null");\r
@@ -523,8 +495,8 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
 \r
                int rank = fauEuTaxon.getRankId();\r
                \r
-               if(logger.isInfoEnabled()) { \r
-                       logger.info("Local taxon name (rank = " + rank + "): " + localString); \r
+               if(logger.isDebugEnabled()) { \r
+                       logger.debug("Local taxon name (rank = " + rank + "): " + localString); \r
                }\r
                \r
                callCount = 0;\r
@@ -536,7 +508,7 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
        \r
        /* Build taxon name */\r
        private boolean setTaxonName(String concatString, FaunaEuropaeaTaxon fauEuTaxon,\r
-                       TaxonBase<?> taxonBase) {\r
+                       TaxonBase<?> taxonBase, FaunaEuropaeaImportConfigurator fauEuConfig) {\r
 \r
                boolean success = true;\r
                \r
@@ -547,14 +519,17 @@ public class FaunaEuropaeaTaxonImport extends FaunaEuropaeaImportBase  {
                String titleCache = buildNameTitleCache(concatString, fauEuTaxon);\r
                zooName.setTitleCache(titleCache);\r
                zooName.setFullTitleCache(titleCache); // TODO: Add reference, NC status\r
+               \r
+               ImportHelper.setOriginalSource(taxonName, fauEuConfig.getSourceReference(), \r
+                               fauEuTaxon.getId(), "TaxonName");\r
 \r
                // Set the complete scientific name in FaunaEuropaeaTaxon,\r
                // including parent(s) parts.\r
 //             fauEuTaxon.setScientificName(concatString);\r
                fauEuTaxon.setNameComplete(true);\r
 \r
-               if (logger.isInfoEnabled()) { \r
-                       logger.info("Name stored: " + concatString); \r
+               if (logger.isDebugEnabled()) { \r
+                       logger.debug("Name stored: " + concatString); \r
                }\r
                return success;\r
        }\r