latest changes in BM import
authorAndreas Müller <a.mueller@bgbm.org>
Tue, 21 Feb 2012 22:44:45 +0000 (22:44 +0000)
committerAndreas Müller <a.mueller@bgbm.org>
Tue, 21 Feb 2012 22:44:45 +0000 (22:44 +0000)
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelImportConfigurator.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelOccurrenceImport.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonNameImport.java

index 7e496251a3dc948c8614d51bce9af23030fad673..44afef2722405fda285ec34c9e62ee953994ef0f 100644 (file)
@@ -67,6 +67,9 @@ public class BerlinModelImportConfigurator extends ImportConfiguratorBase<Berlin
 \r
        private boolean useSingleClassification = false;\r
        \r
+       //occurrences\r
+       private boolean isSplitTdwgCodes = true;\r
+       \r
        /* Max number of records to be saved with one service call */\r
        private int recordsPerTransaction = 1000;\r
 \r
@@ -100,7 +103,6 @@ public class BerlinModelImportConfigurator extends ImportConfiguratorBase<Berlin
        private String occurrenceFilter = null;\r
        private String occurrenceSourceFilter = null;\r
        private String webMarkerFilter = null;\r
-       \r
 \r
        protected void makeIoClassList(){\r
                ioClassList = new Class[]{\r
@@ -657,6 +659,17 @@ public class BerlinModelImportConfigurator extends ImportConfiguratorBase<Berlin
 \r
 \r
 \r
+       public boolean isSplitTdwgCodes() {\r
+               return isSplitTdwgCodes;\r
+       }\r
+       \r
+       \r
+       public void setSplitTdwgCodes(boolean isSplitTdwgCodes) {\r
+               this.isSplitTdwgCodes = isSplitTdwgCodes;\r
+       }\r
+\r
+\r
+\r
 \r
 \r
 \r
index 57a5a4c926f648d2bca76087e91afe562409184b..9ffd0a6bfbf3e8e1cb63516a68e2b8d3cf960dee 100644 (file)
@@ -123,7 +123,7 @@ public class BerlinModelOccurrenceImport  extends BerlinModelImportBase {
                        //for each reference\r
             while (rs.next()){\r
                 \r
-                if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("Facts handled: " + (i-1));}\r
+               if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("Facts handled: " + (i-1));}\r
                 \r
                 int occurrenceId = rs.getInt("OccurrenceId");\r
                 int newTaxonId = rs.getInt("taxonId");\r
@@ -132,30 +132,36 @@ public class BerlinModelOccurrenceImport  extends BerlinModelImportBase {
                 Integer emStatusId = (Integer)rs.getObject("emOccurSumCatId");\r
                 \r
                 try {\r
-                    //status\r
-                     PresenceAbsenceTermBase<?> status = null;\r
-                     String alternativeStatusString = null;\r
-                     if (emStatusId != null){\r
-                       status = BerlinModelTransformer.occStatus2PresenceAbsence(emStatusId);\r
-                     }else{\r
-                        String[] stringArray = new String[]{rs.getString("Native"), rs.getString("Introduced"), rs.getString("Cultivated")};\r
-                        alternativeStatusString = CdmUtils.concat(",", stringArray);\r
-                     }\r
+                       //status\r
+                       PresenceAbsenceTermBase<?> status = null;\r
+                       String alternativeStatusString = null;\r
+                                       if (emStatusId != null){\r
+                                               status = BerlinModelTransformer.occStatus2PresenceAbsence(emStatusId);\r
+                                       }else{\r
+                                               String[] stringArray = new String[]{rs.getString("Native"), rs.getString("Introduced"), rs.getString("Cultivated")};\r
+                                               alternativeStatusString = CdmUtils.concat(",", stringArray);\r
+                                       }\r
                      \r
-                     //Create area list\r
-                     List<NamedArea> areas = new ArrayList<NamedArea>();\r
-                     if (tdwgCodeString != null){\r
-                           String[] tdwgCodes = tdwgCodeString.split(";");\r
-                           for (String tdwgCode : tdwgCodes){\r
-                                 NamedArea area = TdwgArea.getAreaByTdwgAbbreviation(tdwgCode.trim());\r
-                                if (area == null){\r
-                                        area = getOtherAreas(state, emCodeString, tdwgCodeString);\r
-                                }\r
-                                 if (area != null){\r
-                                       areas.add(area);\r
-                                 }\r
-                           }\r
+                                       //Create area list\r
+                                       List<NamedArea> areas = new ArrayList<NamedArea>();\r
+                                       if (tdwgCodeString != null){\r
+                                       \r
+                                               String[] tdwgCodes = new String[]{tdwgCodeString};\r
+                                               if (state.getConfig().isSplitTdwgCodes()){\r
+                                                       tdwgCodes = tdwgCodeString.split(";");\r
+                                               }\r
+                                               \r
+                                               for (String tdwgCode : tdwgCodes){\r
+                                                       NamedArea area = TdwgArea.getAreaByTdwgAbbreviation(tdwgCode.trim());\r
+                                               if (area == null){\r
+                                                       area = getOtherAreas(state, emCodeString, tdwgCodeString);\r
+                                               }\r
+                                               if (area != null){\r
+                                                       areas.add(area);\r
+                                               }\r
+                                       }\r
                      }\r
+                                       \r
                      Reference<?> sourceRef = state.getConfig().getSourceReference();\r
                      //create description(elements)\r
                      TaxonDescription taxonDescription = getTaxonDescription(newTaxonId, oldTaxonId, oldDescription, taxonMap, occurrenceId, sourceRef);\r
@@ -236,6 +242,8 @@ public class BerlinModelOccurrenceImport  extends BerlinModelImportBase {
                        return WaterbodyOrCountry.AZERBAIJANREPUBLICOF();\r
                }else if("TCS-AB;TCS-AD;TCS-GR".equals(tdwg)){\r
                        return WaterbodyOrCountry.GEORGIA();\r
+               \r
+               \r
                }else if("Cc".equals(em)){\r
                        return getNamedArea(state, BerlinModelTransformer.uuidCaucasia, "Caucasia (Ab + Ar + Gg + Rf(CS))", "Euro+Med area 'Caucasia (Ab + Ar + Gg + Rf(CS))'", "Cc", null, null);\r
                }\r
@@ -252,6 +260,8 @@ public class BerlinModelOccurrenceImport  extends BerlinModelImportBase {
                \r
                }else if("SM".equals(em)){\r
                        return getNamedArea(state, BerlinModelTransformer.uuidSerbiaMontenegro, "Serbia & Montenegro", "Euro+Med area 'Serbia & Montenegro'", "SM", NamedAreaType.ADMINISTRATION_AREA(), null);\r
+               }else if("Yu(K)".equals(em)){\r
+                       return TdwgArea.getAreaByTdwgAbbreviation("YUG-KO");\r
                \r
                \r
                //see #2769\r
index 421b2eda9f05dd2ade141082a4e121092317e73a..6983283c3ae61c4ba5cf06330e36ac7668acc2c6 100644 (file)
@@ -165,108 +165,6 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase {
                return null;  //not needed\r
        }\r
 \r
-\r
-//     @Override\r
-//     protected void doInvoke_old(BerlinModelImportState state){\r
-//             logger.info("start make " + getPluralString() + " ...");\r
-//\r
-//             boolean success = true;\r
-//             initializeMappers(state);\r
-//             BerlinModelImportConfigurator config = state.getConfig();\r
-//             Source source = config.getSource();\r
-//\r
-//             String strSelectId = " SELECT Reference.RefId as refId ";\r
-//             String strSelectFull = \r
-//                     " SELECT Reference.* , InReference.RefId as InRefId, InReference.RefCategoryFk as InRefCategoryFk,  " +\r
-//                     " InInReference.RefId as InInRefId, InInReference.RefCategoryFk as InInRefCategoryFk, " +\r
-//                     " InReference.InRefFk AS InRefInRefFk, InInReference.InRefFk AS InInRefInRefFk, RefSource.RefSource " ;\r
-//             String strFrom =  " FROM Reference AS InInReference " +\r
-//                     " RIGHT OUTER JOIN Reference AS InReference ON InInReference.RefId = InReference.InRefFk " + \r
-//                     " RIGHT OUTER JOIN %s ON InReference.RefId = Reference.InRefFk " + \r
-//                     " LEFT OUTER JOIN RefSource ON Reference.RefSourceFk = RefSource.RefSourceId " +\r
-//                     " WHERE (1=1) ";\r
-//             String strWherePartitioned = " AND (Reference.refId IN ("+ ID_LIST_TOKEN + ") ) "; \r
-//             \r
-//             String referenceTable = CdmUtils.Nz(state.getConfig().getReferenceIdTable());\r
-//             referenceTable = referenceTable.isEmpty() ? " Reference"  : referenceTable + " as Reference ";\r
-//             String strIdFrom = String.format(strFrom, referenceTable );\r
-//             \r
-//             \r
-//             //test max number of recursions\r
-//             String strQueryTestMaxRecursion = strSelectId + strIdFrom +  \r
-//                     " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NOT NULL) ";\r
-//             ResultSet testMaxRecursionResultSet = source.getResultSet(strQueryTestMaxRecursion);\r
-//             try {\r
-//                     if (testMaxRecursionResultSet.next() == true){\r
-//                             logger.error("Maximum allowed InReference recursions exceeded in Berlin Model. Maximum recursion level is 2.");\r
-//                             state.setUnsuccessfull();\r
-//                             return;\r
-//                     }\r
-//             } catch (SQLException e1) {\r
-//                     e1.printStackTrace();\r
-//                     logger.error("There are references with more then 2 in-reference recursion. Maximum number of allowed recursions is 2. Records will not be stored.");\r
-//                     success = false;\r
-//             }\r
-//\r
-//             String strSelectIdBase = strSelectId + strIdFrom;\r
-//             \r
-//             String referenceFilter = CdmUtils.Nz(state.getConfig().getReferenceIdTable());\r
-//             if (! referenceFilter.isEmpty()){\r
-//                     referenceFilter = " AND " + referenceFilter + " ";\r
-//             }\r
-//             referenceFilter = "";  //don't use it for now\r
-//             \r
-//             String strIdQueryNoInRef = strSelectIdBase + \r
-//                     " AND (Reference.InRefFk is NULL) " +  referenceFilter;\r
-//             String strIdQuery1InRef = strSelectIdBase + \r
-//                     " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NULL) "  +  referenceFilter;\r
-//             String strIdQuery2InRefs = strSelectIdBase + \r
-//                     " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NULL) "  +  referenceFilter;\r
-//\r
-//             if (config.getDoReferences() == CONCEPT_REFERENCES){\r
-//                     strIdQueryNoInRef += " AND ( Reference.refId IN ( SELECT ptRefFk FROM PTaxon) ) " + referenceFilter;\r
-//             }\r
-//\r
-//             String strRecordQuery = strSelectFull + String.format(strFrom, " Reference ") + strWherePartitioned;\r
-//             \r
-//             int recordsPerTransaction = config.getRecordsPerTransaction();\r
-//             try{\r
-//                     //NoInRefs\r
-//                     ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQueryNoInRef, strRecordQuery, recordsPerTransaction);\r
-//                     while (partitioner.nextPartition()){\r
-//                             partitioner.doPartition(this, state);\r
-//                     }\r
-//                     logger.info("end make references with no in-references ... " + getSuccessString(success));\r
-//\r
-//                     if (config.getDoReferences() == ALL || config.getDoReferences() == NOMENCLATURAL){\r
-//\r
-//                             //1InRef\r
-//                             partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery1InRef, strRecordQuery, recordsPerTransaction);\r
-//                             while (partitioner.nextPartition()){\r
-//                                     partitioner.doPartition(this, state);\r
-//                             }\r
-//                             logger.info("end make references with no 1 in-reference ... " + getSuccessString(success));\r
-//     \r
-//                             //2InRefs\r
-//                             partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery2InRefs, strRecordQuery, recordsPerTransaction);\r
-//                             while (partitioner.nextPartition()){\r
-//                                     partitioner.doPartition(this, state);\r
-//                             }\r
-//                             logger.info("end make references with no 2 in-reference ... " + getSuccessString(success));\r
-//                     }\r
-//\r
-//             } catch (SQLException e) {\r
-//                     logger.error("SQLException:" +  e);\r
-//                     state.setUnsuccessfull();\r
-//                     return;\r
-//             }\r
-//             logger.info("end make " + getPluralString() + " ... " + getSuccessString(success));\r
-//             if (! success){\r
-//                     state.setUnsuccessfull();\r
-//             }\r
-//             return;\r
-//     }\r
-\r
        @Override\r
        protected void doInvoke(BerlinModelImportState state){\r
                logger.info("start make " + getPluralString() + " ...");\r
@@ -289,23 +187,6 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase {
                referenceTable = referenceTable.isEmpty() ? " Reference"  : referenceTable + " as Reference ";\r
                String strIdFrom = String.format(strFrom, referenceTable );\r
                \r
-               \r
-               //test max number of recursions\r
-//             String strQueryTestMaxRecursion = strSelectId + strIdFrom +  \r
-//                     " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NOT NULL) ";\r
-//             ResultSet testMaxRecursionResultSet = source.getResultSet(strQueryTestMaxRecursion);\r
-//             try {\r
-//                     if (testMaxRecursionResultSet.next() == true){\r
-//                             logger.error("Maximum allowed InReference recursions exceeded in Berlin Model. Maximum recursion level is 2.");\r
-//                             state.setUnsuccessfull();\r
-//                             return;\r
-//                     }\r
-//             } catch (SQLException e1) {\r
-//                     e1.printStackTrace();\r
-//                     logger.error("There are references with more then 2 in-reference recursion. Maximum number of allowed recursions is 2. Records will not be stored.");\r
-//                     success = false;\r
-//             }\r
-\r
                String strSelectIdBase = strSelectId + strIdFrom;\r
                \r
                String referenceFilter = CdmUtils.Nz(state.getConfig().getReferenceIdTable());\r
@@ -462,7 +343,7 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase {
                                                        inRefExists |= (inRef != null);\r
                                                }\r
                                                if (thisBiblioRef != null){\r
-                                                       Reference<?> inRef = (biblioInReference != null)? biblioInReference : nomInReference;\r
+                                                       Reference<?> inRef = (biblioInReference != null)? nomInReference : biblioInReference ;\r
                                                        thisBiblioRef.setInReference(inRef);\r
                                                        biblioRefToSave.put(refId, thisBiblioRef);\r
                                                        //remember that an in reference exists\r
index dce7c57f02e0d2156d5b274dfb4ae735e6c55902..a149bd86c67da1d07e0b9c8b7f9d0540f8f247cc 100644 (file)
@@ -114,7 +114,7 @@ public class BerlinModelTaxonNameImport extends BerlinModelImportBase {
                 " WHERE name.nameId IN ("+ID_LIST_TOKEN+") ";\r
                                        //strQuery += " AND RefDetail.PreliminaryFlag = 1 ";\r
                                        //strQuery += " AND Name.Created_When > '03.03.2004' ";\r
-               return strRecordQuery;\r
+               return strRecordQuery +  "";\r
        }\r
 \r
 \r
@@ -152,8 +152,8 @@ public class BerlinModelTaxonNameImport extends BerlinModelImportBase {
                                        boolean useUnknownRank = true;\r
                                        Rank rank = BerlinModelTransformer.rankId2Rank(rs, useUnknownRank, config.isSwitchSpeciesGroup());\r
                                        \r
-                                       if (rank.equals(Rank.UNKNOWN_RANK())){\r
-                                               rank = handleProlesAndRace(state, rs);\r
+                                       if (rank == null || rank.equals(Rank.UNKNOWN_RANK()) || rank.equals(Rank.INFRASPECIFICTAXON())){\r
+                                               rank = handleProlesAndRace(state, rs, rank);\r
                                        }\r
                                        \r
                                        if (rank.getId() == 0){\r
@@ -294,16 +294,17 @@ public class BerlinModelTaxonNameImport extends BerlinModelImportBase {
        }\r
 \r
 \r
-       private Rank handleProlesAndRace(BerlinModelImportState state, ResultSet rs) throws SQLException {\r
+       private Rank handleProlesAndRace(BerlinModelImportState state, ResultSet rs, Rank rank) throws SQLException {\r
                Rank result;\r
                String rankAbbrev = rs.getString("RankAbbrev");\r
                String rankStr = rs.getString("Rank");\r
                if (CdmUtils.nullSafeEqual(rankAbbrev, "prol.") ){\r
-                       result = getRank(state, BerlinModelTransformer.uuidRankProles, "Rank Proles", "Rank Proles", "prol.", Rank.SPECIES().getVocabulary());\r
+                       result = getRank(state, BerlinModelTransformer.uuidRankProles, rankStr, "Rank Proles", rankAbbrev, Rank.SPECIES().getVocabulary());\r
                }else if(CdmUtils.nullSafeEqual(rankAbbrev, "race")){\r
-                       result = getRank(state, BerlinModelTransformer.uuidRankRace, "Race", "Rank Race", "race", Rank.SPECIES().getVocabulary());\r
+                       result = getRank(state, BerlinModelTransformer.uuidRankRace, rankStr, "Rank Race", rankAbbrev, Rank.SPECIES().getVocabulary());\r
                }else{\r
-                       result = Rank.UNKNOWN_RANK();\r
+                       result = rank;\r
+                       logger.warn("Unhandled rank: " + rankAbbrev);\r
                }\r
                return result;\r
        }\r