ref #7341 validate common names with nameUseInSourceFk = -1
[cdmlib-apps.git] / app-import / src / main / java / eu / etaxonomy / cdm / io / berlinModel / in / BerlinModelCommonNamesImport.java
index 1910b0e5faa668df64ad08599ae6acdae8261bdd..d015d138d83e00839885a42428df9f2f2ecd3cd0 100644 (file)
@@ -32,8 +32,6 @@ import eu.etaxonomy.cdm.io.common.IOValidator;
 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;\r
 import eu.etaxonomy.cdm.io.common.Source;\r
 import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;\r
-import eu.etaxonomy.cdm.model.common.Annotation;\r
-import eu.etaxonomy.cdm.model.common.AnnotationType;\r
 import eu.etaxonomy.cdm.model.common.CdmBase;\r
 import eu.etaxonomy.cdm.model.common.Extension;\r
 import eu.etaxonomy.cdm.model.common.ExtensionType;\r
@@ -41,12 +39,14 @@ import eu.etaxonomy.cdm.model.common.Language;
 import eu.etaxonomy.cdm.model.common.Marker;\r
 import eu.etaxonomy.cdm.model.common.MarkerType;\r
 import eu.etaxonomy.cdm.model.common.Representation;\r
+import eu.etaxonomy.cdm.model.common.TermType;\r
+import eu.etaxonomy.cdm.model.common.TermVocabulary;\r
 import eu.etaxonomy.cdm.model.description.CommonTaxonName;\r
 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;\r
 import eu.etaxonomy.cdm.model.description.TaxonDescription;\r
 import eu.etaxonomy.cdm.model.location.Country;\r
 import eu.etaxonomy.cdm.model.location.NamedArea;\r
-import eu.etaxonomy.cdm.model.name.TaxonNameBase;\r
+import eu.etaxonomy.cdm.model.name.TaxonName;\r
 import eu.etaxonomy.cdm.model.reference.Reference;\r
 import eu.etaxonomy.cdm.model.taxon.Taxon;\r
 import eu.etaxonomy.cdm.model.taxon.TaxonBase;\r
@@ -54,15 +54,18 @@ import eu.etaxonomy.cdm.model.taxon.TaxonBase;
 /**\r
  *\r
  * @author a.mueller\r
- * @created 20.03.2008\r
+ * @since 20.03.2008\r
  */\r
 @Component\r
 public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {\r
-       private static final Logger logger = Logger.getLogger(BerlinModelCommonNamesImport.class);\r
+    private static final long serialVersionUID = -8921948187177864321L;\r
+\r
+    private static final Logger logger = Logger.getLogger(BerlinModelCommonNamesImport.class);\r
 \r
        public static final UUID REFERENCE_LANGUAGE_ISO639_2_UUID = UUID.fromString("40c4f8dd-3d9c-44a4-b77a-76e137a89a5f");\r
        public static final UUID REFERENCE_LANGUAGE_STRING_UUID = UUID.fromString("2a1b678f-c27d-48c1-b43e-98fd0d426305");\r
-       public static final UUID STATUS_ANNOTATION_UUID = UUID.fromString("e3f7b80a-1286-458d-812c-5e818f731968");\r
+       public static final UUID COMMONNAME_STATUS_RECOMMENDED_UUID = UUID.fromString("e3f7b80a-1286-458d-812c-5e818f731968");\r
+       public static final UUID COMMONNAME_STATUS_SYNONYM_UUID = UUID.fromString("169b2d97-a706-49de-b28b-c67f0ee6764b");\r
 \r
        public static final String NAMESPACE = "common name";\r
 \r
@@ -72,7 +75,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
 \r
 \r
        //map that stores the regions (named areas) and makes them accessible via the regionFk\r
-       private Map<String, NamedArea> regionMap = new HashMap<String, NamedArea>();\r
+       private Map<String, NamedArea> regionMap = new HashMap<>();\r
 \r
        public BerlinModelCommonNamesImport(){\r
                super(dbTableName, pluralString);\r
@@ -81,9 +84,10 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
        @Override\r
        protected String getIdQuery(BerlinModelImportState state) {\r
                String result = " SELECT CommonNameId FROM emCommonName WHERE (1=1) ";\r
-               if (StringUtils.isNotBlank(state.getConfig().getCommonNameFilter())){\r
+               if (isNotBlank(state.getConfig().getCommonNameFilter())){\r
                        result += " AND " + state.getConfig().getCommonNameFilter();\r
                }\r
+               result += " ORDER BY CommonNameId ";\r
 \r
                return result;\r
        }\r
@@ -92,21 +96,30 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
        protected String getRecordQuery(BerlinModelImportConfigurator config) {\r
                String recordQuery = "";\r
                recordQuery =\r
-                               " SELECT     cn.CommonNameId, cn.CommonName, PTaxon.RIdentifier AS taxonId, cn.PTNameFk, cn.RefFk AS refId, cn.Status, cn.RegionFks, cn.MisNameRefFk, " +\r
-                                              "               cn.NameInSourceFk, cn.Created_When, cn.Updated_When, cn.Created_Who, cn.Updated_Who, cn.Note AS Notes, languageCommonName.Language, " +\r
-                                              "               languageCommonName.LanguageOriginal, languageCommonName.ISO639_1, languageCommonName.ISO639_2,   " +\r
-                                              "               emLanguageReference.RefFk AS languageRefRefFk, emLanguageReference.ReferenceShort, emLanguageReference.ReferenceLong,  " +\r
-                                              "               emLanguageReference.LanguageFk, languageReferenceLanguage.Language AS refLanguage, languageReferenceLanguage.ISO639_2 AS refLanguageIso639_2,  "+\r
-                                              "               misappliedTaxon.RIdentifier AS misappliedTaxonId " +\r
-                                       " FROM         PTaxon AS misappliedTaxon RIGHT OUTER JOIN " +\r
-                                           "                  emLanguage AS languageReferenceLanguage RIGHT OUTER JOIN " +\r
-                                                      "       emLanguageReference ON languageReferenceLanguage.LanguageId = emLanguageReference.LanguageFk RIGHT OUTER JOIN " +\r
-                                                      "       emCommonName AS cn INNER JOIN " +\r
-                                                      "       PTaxon ON cn.PTNameFk = PTaxon.PTNameFk AND cn.PTRefFk = PTaxon.PTRefFk ON  " +\r
-                                                      "       emLanguageReference.ReferenceId = cn.LanguageRefFk LEFT OUTER JOIN " +\r
-                                                       "      emLanguage AS languageCommonName ON cn.LanguageFk = languageCommonName.LanguageId ON misappliedTaxon.PTNameFk = cn.NameInSourceFk AND  " +\r
-                                                       "      misappliedTaxon.PTRefFk = cn.MisNameRefFk " +\r
-                       " WHERE cn.CommonNameId IN (" + ID_LIST_TOKEN + ")";\r
+                               " SELECT rel.RelPTaxonId, rel.RelQualifierFk, acc.RIdentifier accTaxonId, factTaxon.RIdentifier factTaxonId, accName.NameId, f.FactId, " +\r
+                                          "       cn.CommonNameId, cn.CommonName, tax.RIdentifier AS taxonId, cn.PTNameFk, cn.RefFk AS refId, cn.Status, cn.RegionFks, cn.MisNameRefFk, " +\r
+                                              "       cn.NameInSourceFk, cn.Created_When, cn.Updated_When, cn.Created_Who, cn.Updated_Who, cn.Note AS Notes, languageCommonName.Language, " +\r
+                                              "       languageCommonName.LanguageOriginal, languageCommonName.ISO639_1, languageCommonName.ISO639_2,   " +\r
+                                              "       emLanguageReference.RefFk AS languageRefRefFk, emLanguageReference.ReferenceShort, emLanguageReference.ReferenceLong,  " +\r
+                                              "       emLanguageReference.LanguageFk, languageReferenceLanguage.Language AS refLanguage, languageReferenceLanguage.ISO639_2 AS refLanguageIso639_2,  "+\r
+                                              "       misappliedTaxon.RIdentifier AS misappliedTaxonId " +\r
+                                 " FROM  PTaxon AS misappliedTaxon RIGHT OUTER JOIN " +\r
+                                              "      emLanguage AS languageReferenceLanguage RIGHT OUTER JOIN " +\r
+                                      "      emLanguageReference ON languageReferenceLanguage.LanguageId = emLanguageReference.LanguageFk RIGHT OUTER JOIN " +\r
+                                      "      emCommonName AS cn INNER JOIN " +\r
+                                      "      PTaxon AS tax ON cn.PTNameFk = tax.PTNameFk AND cn.PTRefFk = tax.PTRefFk ON  " +\r
+                                      "      emLanguageReference.ReferenceId = cn.LanguageRefFk LEFT OUTER JOIN " +\r
+                                      "      emLanguage AS languageCommonName ON cn.LanguageFk = languageCommonName.LanguageId ON misappliedTaxon.PTNameFk = cn.NameInSourceFk AND  " +\r
+                                      "      misappliedTaxon.PTRefFk = cn.MisNameRefFk " +\r
+\r
+                            "     LEFT OUTER JOIN Fact f ON cn.CommonNameId = f.ExtensionFk " +\r
+                            "     LEFT OUTER JOIN PTaxon factTaxon ON factTaxon.PTNameFk = f.PTNameFk AND factTaxon.PTRefFk = f.PTRefFk " +\r
+                            "     LEFT OUTER JOIN RelPTaxon rel ON rel.PTNameFk1 = tax.PTNameFk AND rel.PTRefFk1 = tax.PTRefFk AND rel.RelQualifierFk IN (2,6,7) " +\r
+                         "     LEFT OUTER JOIN PTaxon acc ON rel.PTNameFk2 = acc.PTNameFk AND rel.PTRefFk2 = acc.PTRefFk " +\r
+                         "     LEFT OUTER JOIN Name accName ON accName.NameId = acc.PTNameFk " +\r
+                               " WHERE cn.CommonNameId IN (" + ID_LIST_TOKEN + ") " +\r
+                               " ORDER BY cn.CommonNameId ";\r
+\r
                return recordQuery;\r
        }\r
 \r
@@ -129,7 +142,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
         */\r
        private void makeRegions(BerlinModelImportState state) {\r
                try {\r
-                       SortedSet<Integer> regionFks = new TreeSet<Integer>();\r
+                       SortedSet<Integer> regionFks = new TreeSet<>();\r
                        Source source = state.getConfig().getSource();\r
 \r
                        //fill set with all regionFk from emCommonName.regionFks\r
@@ -159,23 +172,29 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
        public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state)  {\r
                boolean success = true ;\r
 \r
-               Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();\r
-               Map<String, Taxon> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);\r
-               Map<String, TaxonNameBase> taxonNameMap = partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);\r
+               Set<TaxonBase> taxaToSave = new HashSet<>();\r
+               @SuppressWarnings("unchecked")\r
+        Map<String, Taxon> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);\r
+               @SuppressWarnings("unchecked")\r
+        Map<String, TaxonName> taxonNameMap = partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);\r
+               @SuppressWarnings("unchecked")\r
+        Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);\r
 \r
-               Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);\r
-\r
-               Map<String, Language> iso6392Map = new HashMap<String, Language>();\r
+               Map<String, Language> iso6392Map = new HashMap<>();\r
 \r
        //      logger.warn("MisappliedNameRefFk  not yet implemented for Common Names");\r
 \r
                ResultSet rs = partitioner.getResultSet();\r
+               Integer lastCommonNameId = null;\r
                try{\r
                        while (rs.next()){\r
 \r
                                //create TaxonName element\r
                                Integer commonNameId = rs.getInt("CommonNameId");\r
                                int taxonId = rs.getInt("taxonId");\r
+                               Integer factTaxonId = nullSafeInt(rs, "factTaxonId");\r
+                               Integer accTaxonId = nullSafeInt(rs, "accTaxonId");  //if common name is related to synonym this is the accepted taxon id\r
+\r
                                Integer refId = nullSafeInt(rs, "refId");\r
 //                             Integer ptNameFk = nullSafeInt(rs,"PTNameFk");\r
                                String commonNameString = rs.getString("CommonName");\r
@@ -191,9 +210,15 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                Integer nameInSourceFk = nullSafeInt( rs, "NameInSourceFk");\r
                                Integer misappliedTaxonId = nullSafeInt( rs, "misappliedTaxonId");\r
 \r
+                               if (commonNameId == lastCommonNameId){\r
+                                   logger.warn("CommonNameId >1 times in query. This may happen due to LEFT JOINS to fact and/or accepted taxon and e.g. multiple taxon relationships. 2018-04-01 no such double relation existed in E+M. ");\r
+                               }else{\r
+                                   lastCommonNameId = commonNameId;\r
+                               }\r
+\r
                                //regions\r
                                String regionFks  = rs.getString("RegionFks");\r
-                               String[] regionFkSplit = regionFks.split(",");\r
+                               String[] regionFkSplit = (regionFks==null)? new String[0] : regionFks.split(",");\r
 \r
                                //commonNameString\r
                                if (isBlank(commonNameString)){\r
@@ -208,18 +233,29 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                if (taxonBase == null){\r
                                        logger.warn("Taxon (" + taxonId + ") could not be found. Common name " + commonNameString + "(" + commonNameId + ") not imported");\r
                                        continue;\r
-                               }else if (! taxonBase.isInstanceOf(Taxon.class)){\r
-                                       logger.warn("taxon (" + taxonId + ") is not accepted. Can't import common name " +  commonNameId);\r
-                                       continue;\r
+                               }else if (taxonBase.isInstanceOf(Taxon.class)){\r
+                                   taxon = CdmBase.deproxy(taxonBase, Taxon.class);\r
+                                   if (factTaxonId != null && !factTaxonId.equals(taxonId)){\r
+                                       logger.warn("Fact taxon ("+factTaxonId+") for common name "+commonNameId+" differs from common name taxon " + taxonId);\r
+                                   }\r
                                }else{\r
-                                       taxon = CdmBase.deproxy(taxonBase, Taxon.class);\r
+                                   Taxon factTaxon = null;\r
+                                   if (factTaxonId != null && factTaxonId.equals(accTaxonId)){\r
+                                       factTaxon = taxonMap.get(String.valueOf(factTaxonId));\r
+                                   }\r
+                                   if (factTaxon != null){\r
+                                       taxon = factTaxon;\r
+                                   }else{\r
+                                       logger.warn("taxon (" + taxonId + ") is not accepted. Can't import common name " +  commonNameId + ". FactTaxonId= " +  factTaxonId + "; accTaxonId = " + accTaxonId);\r
+                                       continue;\r
+                                   }\r
                                }\r
 \r
                                //Language\r
                                Language language = getAndHandleLanguage(iso6392Map, iso639_2, iso639_1, languageString, originalLanguageString, state);\r
 \r
                                //CommonTaxonName\r
-                               List<CommonTaxonName> commonTaxonNames = new ArrayList<CommonTaxonName>();\r
+                               List<CommonTaxonName> commonTaxonNames = new ArrayList<>();\r
                                for (String regionFk : regionFkSplit){ //\r
                                        CommonTaxonName commonTaxonName;\r
                                        if (commonTaxonNames.size() == 0){\r
@@ -231,7 +267,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                        regionFk = regionFk.trim();\r
                                        NamedArea area = regionMap.get(regionFk);\r
                                        if (area == null){\r
-                                               if (regionFkSplit.length > 1 && StringUtils.isNotBlank(regionFk)){\r
+                                               if (regionFkSplit.length > 1 && isNotBlank(regionFk)){\r
                                                        logger.warn("Area for " + regionFk + " not defined in regionMap.");\r
                                                }else{\r
                                                        //no region is defined\r
@@ -249,7 +285,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                        if (languageRefRefFk == null){\r
                                                languageRefRefFk = refId;\r
                                        }else{\r
-                                               logger.warn("CommonName.RefFk (" + CdmUtils.Nz(refId) + ") and LanguageReference.RefFk " + (languageRefRefFk==null? "null" : languageRefRefFk)  + " are not equal. I will import only languageReference.RefFk");\r
+                                               logger.warn("CommonName.RefFk (" + CdmUtils.Nz(refId) + ") and LanguageReference.RefFk " + languageRefRefFk  + " are not equal. I will import only languageReference.RefFk");\r
                                        }\r
                                }\r
 \r
@@ -260,13 +296,15 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                String microCitation = null;\r
                                String originalNameString = null;\r
 \r
-                               TaxonNameBase<?,?> nameUsedInSource = taxonNameMap.get(String.valueOf(nameInSourceFk));\r
+                               TaxonName nameUsedInSource = taxonNameMap.get(String.valueOf(nameInSourceFk));\r
                                if (nameInSourceFk != null && nameUsedInSource == null){\r
-                                       logger.warn("Name used in source (" + nameInSourceFk + ") was not found for common name " + commonNameId);\r
+                                       if (nameInSourceFk != -1 || !state.getConfig().isEuroMed()){\r
+                                           logger.warn("Name used in source (" + nameInSourceFk + ") was not found for common name " + commonNameId);\r
+                                       }\r
                                }\r
-                               DescriptionElementSource source = DescriptionElementSource.NewPrimarySourceInstance(reference, microCitation, nameUsedInSource, originalNameString);\r
                                for (CommonTaxonName commonTaxonName : commonTaxonNames){\r
-                                       commonTaxonName.addSource(source);\r
+                                   DescriptionElementSource source = DescriptionElementSource.NewPrimarySourceInstance(reference, microCitation, nameUsedInSource, originalNameString);\r
+                       commonTaxonName.addSource(source);\r
                                }\r
 \r
 \r
@@ -300,7 +338,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                        }\r
                                        if (misappliedNameTaxon != null){\r
 \r
-                                               if (! taxon.getMisappliedNames().contains(misappliedNameTaxon)){\r
+                                               if (! taxon.getMisappliedNames(false).contains(misappliedNameTaxon)){\r
                                                        taxon.addMisappliedName(misappliedNameTaxon,state.getTransactionalSourceReference(), null);\r
                                                        logger.warn("Misapplied name for common name was not found related to the accepted taxon. Created new relationship. CommonNameId: " + commonNameId);\r
                                                }\r
@@ -311,7 +349,8 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                                        misappliedNameDescription.addElement(commonNameClone);\r
                                                }\r
                                        }else{\r
-                                               logger.warn("Misapplied name is null for common name " + commonNameId);\r
+                                               //wird schon oben gelogged\r
+                                           //logger.warn("Misapplied name is null for common name " + commonNameId);\r
                                        }\r
 \r
                                }\r
@@ -334,10 +373,26 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
 \r
                                //status\r
                                if (isNotBlank(status)){\r
-                                       AnnotationType statusAnnotationType = getAnnotationType( state, STATUS_ANNOTATION_UUID, "status","The status of this object","status", null);\r
-                                       for (CommonTaxonName commonTaxonName : commonTaxonNames){\r
-                                               Annotation annotation = Annotation.NewInstance(status, statusAnnotationType, Language.DEFAULT());\r
-                                               commonTaxonName.addAnnotation(annotation);\r
+                                   TermVocabulary<MarkerType> markerTypeVoc = getVocabulary(TermType.MarkerType, BerlinModelTransformer.uuidVocEMMarkerType,\r
+                                           "Euro+Med marker type vocabulary", "E+M marker types", null, null, false, MarkerType.COMPLETE());\r
+                                   MarkerType recommendedMarkerType = getMarkerType( state, COMMONNAME_STATUS_RECOMMENDED_UUID, "recommended","If the common name has the status recommended (see also status 'synonym', if none of them is true the default status is 'unassessed')",\r
+                                           "recommended", markerTypeVoc);\r
+                                       MarkerType synonymMarkerType = getMarkerType( state, COMMONNAME_STATUS_SYNONYM_UUID, "synonym","If the common name has the status synonym (see also status 'recommended', if none of them is true the default status is 'unassessed')",\r
+                                               "synonym", markerTypeVoc);\r
+                    for (CommonTaxonName commonTaxonName : commonTaxonNames){\r
+                        Marker marker = null;\r
+                        if (status.equals("recommended")){\r
+                                                   marker = Marker.NewInstance(recommendedMarkerType, true);\r
+                        }else if (status.equals("synonym")){\r
+                            marker = Marker.NewInstance(synonymMarkerType, true);\r
+                        }else if (status.equals("unassessed")){\r
+                            //do nothing\r
+                        }else{\r
+                                                   logger.warn("Unknown common name status: " + status);\r
+                                               }\r
+                        if (marker != null){\r
+                            commonTaxonName.addMarker(marker);\r
+                        }\r
                                        }\r
                                }\r
 \r
@@ -354,7 +409,9 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                        return false;\r
                } catch (ClassCastException e) {\r
                        e.printStackTrace();\r
-               }\r
+               } catch (Exception e) {\r
+            throw e;\r
+        }\r
 \r
                //      logger.info( i + " names handled");\r
                getTaxonService().save(taxaToSave);\r
@@ -503,11 +560,11 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                NamedArea newArea = getNamedArea(state, null, region, "Language region '" + region + "'", null, null, null);\r
 //                             getTermService().save(newArea);\r
                                regionMap.put(String.valueOf(regionId), newArea);\r
-                               logger.warn("Found new area: " +  region);\r
+                               logger.info("Found new area: " +  region);\r
                        }else if (splitRegion.length == 2){\r
                                String emCode = splitRegion[1].trim();\r
                                String tdwgCode = emTdwgMap.get(emCode);\r
-                               if (StringUtils.isNotBlank(tdwgCode) ){\r
+                               if (isNotBlank(tdwgCode) ){\r
                                        NamedArea tdwgArea = getNamedArea(state, tdwgCode);\r
                                        regionMap.put(String.valueOf(regionId), tdwgArea);\r
                                }else {\r
@@ -543,7 +600,12 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                        area = Country.RUSSIANFEDERATION();\r
                }else if (tdwgCode.equalsIgnoreCase("Gg")){\r
                        area = Country.GEORGIA();\r
-               }else{\r
+               }else if (tdwgCode.equalsIgnoreCase("SM")){\r
+            area = getNamedArea(state, BerlinModelTransformer.uuidSM , "Serbia & Montenegro", "Serbia & Montenegro", "SM", null, null);\r
+            getTermService().saveOrUpdate(area);\r
+        }else if (tdwgCode.equalsIgnoreCase("Tu")){\r
+            area = Country.TURKEYREPUBLICOF();\r
+        }else{\r
                        area = TdwgAreaProvider.getAreaByTdwgAbbreviation(tdwgCode);\r
                }\r
                if (area == null){\r
@@ -575,18 +637,19 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
        private Map<String, String> getEmTdwgMap(Source source) throws SQLException {\r
                String sql;\r
                ResultSet rs;\r
-               Map<String, String> emTdwgMap = new HashMap<String, String>();\r
+               Map<String, String> emTdwgMap = new HashMap<>();\r
                sql = " SELECT EmCode, TDWGCode FROM emArea ";\r
                rs = source.getResultSet(sql);\r
                while (rs.next()){\r
                        String emCode = rs.getString("EMCode");\r
                        String TDWGCode = rs.getString("TDWGCode");\r
-                       if (StringUtils.isNotBlank(emCode) ){\r
+                       if (isNotBlank(emCode) ){\r
                                emCode = emCode.trim();\r
                                if (emCode.equalsIgnoreCase("Ab") || emCode.equalsIgnoreCase("Rf")||\r
-                                               emCode.equalsIgnoreCase("Uk") || emCode.equalsIgnoreCase("Gg")){\r
+                                               emCode.equalsIgnoreCase("Uk") || emCode.equalsIgnoreCase("Gg")\r
+                                               || emCode.equalsIgnoreCase("SM") || emCode.equalsIgnoreCase("Tu")){\r
                                        emTdwgMap.put(emCode, emCode);\r
-                               }else if (StringUtils.isNotBlank(TDWGCode)){\r
+                               }else if (isNotBlank(TDWGCode)){\r
                                        emTdwgMap.put(emCode, TDWGCode.trim());\r
                                }\r
                        }\r
@@ -597,6 +660,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                emTdwgMap.put("Uk / Uk(U)", "Uk");\r
                emTdwgMap.put("Ar / Ar(A)", "TCS-AR");\r
                emTdwgMap.put("Hs / Hs(S)", "SPA-SP");\r
+               emTdwgMap.put("Hb / Hb(E)", "IRE-IR");\r
 \r
                return emTdwgMap;\r
        }\r
@@ -625,16 +689,17 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                String nameSpace;\r
                Class<?> cdmClass;\r
                Set<String> idSet;\r
-               Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();\r
+               Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();\r
 \r
                String pos = "0";\r
                try{\r
-                       Set<String> taxonIdSet = new HashSet<String>();\r
-                       Set<String> nameIdSet = new HashSet<String>();\r
-                       Set<String> referenceIdSet = new HashSet<String>();\r
+                       Set<String> taxonIdSet = new HashSet<>();\r
+                       Set<String> nameIdSet = new HashSet<>();\r
+                       Set<String> referenceIdSet = new HashSet<>();\r
                        while (rs.next()){\r
                                handleForeignKey(rs, taxonIdSet, "taxonId");\r
-                               handleForeignKey(rs, taxonIdSet, "misappliedTaxonId");\r
+                               handleForeignKey(rs, taxonIdSet, "factTaxonId");\r
+                handleForeignKey(rs, taxonIdSet, "misappliedTaxonId");\r
                                handleForeignKey(rs, referenceIdSet, "refId");\r
                                handleForeignKey(rs, referenceIdSet, "languageRefRefFk");\r
                                handleForeignKey(rs, nameIdSet, "NameInSourceFk");\r
@@ -644,23 +709,26 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
 \r
                        //name map\r
                        nameSpace = BerlinModelTaxonNameImport.NAMESPACE;\r
-                       cdmClass = TaxonNameBase.class;\r
+                       cdmClass = TaxonName.class;\r
                        idSet = nameIdSet;\r
-                       Map<String, TaxonNameBase<?,?>> nameMap = (Map<String, TaxonNameBase<?,?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       @SuppressWarnings("unchecked")\r
+            Map<String, TaxonName> nameMap = (Map<String, TaxonName>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
                        result.put(nameSpace, nameMap);\r
 \r
                        //taxon map\r
                        nameSpace = BerlinModelTaxonImport.NAMESPACE;\r
                        cdmClass = TaxonBase.class;\r
                        idSet = taxonIdSet;\r
-                       Map<String, TaxonBase<?>> taxonMap = (Map<String, TaxonBase<?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       @SuppressWarnings("unchecked")\r
+            Map<String, TaxonBase<?>> taxonMap = (Map<String, TaxonBase<?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
                        result.put(nameSpace, taxonMap);\r
 \r
                        //reference map\r
                        nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;\r
                        cdmClass = Reference.class;\r
                        idSet = referenceIdSet;\r
-                       Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       @SuppressWarnings("unchecked")\r
+            Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
                        result.put(nameSpace, referenceMap);\r
                        // TODO remove if problem with duplicate DescElement_Annot id is solved\r
                } catch (SQLException e) {\r