Some updates to Berlin MOdel import validation
[cdmlib.git] / cdmlib-io / src / main / java / eu / etaxonomy / cdm / io / berlinModel / in / BerlinModelCommonNamesImport.java
index e3276919e666c3b98b79768d227eb9a7651bf9fc..46451211fe387b269dff1adbf7e550695cd26739 100644 (file)
@@ -31,10 +31,10 @@ import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelCommonNamesImpor
 import eu.etaxonomy.cdm.io.common.IOValidator;\r
 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;\r
 import eu.etaxonomy.cdm.io.common.Source;\r
+import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;\r
 import eu.etaxonomy.cdm.model.common.Annotation;\r
 import eu.etaxonomy.cdm.model.common.AnnotationType;\r
 import eu.etaxonomy.cdm.model.common.CdmBase;\r
-import eu.etaxonomy.cdm.model.common.DescriptionElementSource;\r
 import eu.etaxonomy.cdm.model.common.Extension;\r
 import eu.etaxonomy.cdm.model.common.ExtensionType;\r
 import eu.etaxonomy.cdm.model.common.Language;\r
@@ -42,10 +42,10 @@ import eu.etaxonomy.cdm.model.common.Marker;
 import eu.etaxonomy.cdm.model.common.MarkerType;\r
 import eu.etaxonomy.cdm.model.common.Representation;\r
 import eu.etaxonomy.cdm.model.description.CommonTaxonName;\r
+import eu.etaxonomy.cdm.model.description.DescriptionElementSource;\r
 import eu.etaxonomy.cdm.model.description.TaxonDescription;\r
+import eu.etaxonomy.cdm.model.location.Country;\r
 import eu.etaxonomy.cdm.model.location.NamedArea;\r
-import eu.etaxonomy.cdm.model.location.TdwgArea;\r
-import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;\r
 import eu.etaxonomy.cdm.model.name.TaxonNameBase;\r
 import eu.etaxonomy.cdm.model.reference.Reference;\r
 import eu.etaxonomy.cdm.model.taxon.Taxon;\r
@@ -55,7 +55,6 @@ import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  * \r
  * @author a.mueller\r
  * @created 20.03.2008\r
- * @version 1.0\r
  */\r
 @Component\r
 public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {\r
@@ -75,14 +74,10 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
        //map that stores the regions (named areas) and makes them accessible via the regionFk\r
        private Map<String, NamedArea> regionMap = new HashMap<String, NamedArea>();\r
 \r
-       \r
-\r
        public BerlinModelCommonNamesImport(){\r
                super(dbTableName, pluralString);\r
        }\r
        \r
-       \r
-\r
        @Override\r
        protected String getIdQuery(BerlinModelImportState state) {\r
                String result = " SELECT CommonNameId FROM emCommonName WHERE (1=1) ";\r
@@ -93,11 +88,6 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                return result;\r
        }\r
 \r
-\r
-\r
-       /* (non-Javadoc)\r
-        * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)\r
-        */\r
        @Override\r
        protected String getRecordQuery(BerlinModelImportConfigurator config) {\r
                String recordQuery = "";\r
@@ -120,8 +110,6 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                return recordQuery;\r
        }\r
        \r
-       \r
-\r
        @Override\r
        protected void doInvoke(BerlinModelImportState state) {\r
                try {\r
@@ -167,18 +155,15 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
        }\r
 \r
 \r
-       /* (non-Javadoc)\r
-        * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)\r
-        */\r
+       @Override\r
        public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state)  {\r
                boolean success = true ;\r
-               BerlinModelImportConfigurator config = state.getConfig();\r
+               \r
                Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();\r
                Map<String, Taxon> taxonMap = (Map<String, Taxon>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);\r
                Map<String, TaxonNameBase> taxonNameMap = (Map<String, TaxonNameBase>) partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);\r
                \r
-               Map<String, Reference> biblioRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE);\r
-               Map<String, Reference> nomRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE);\r
+               Map<String, Reference> refMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);\r
                \r
                Map<String, Language> iso6392Map = new HashMap<String, Language>();\r
                \r
@@ -211,7 +196,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                String[] regionFkSplit = regionFks.split(",");\r
                                \r
                                //commonNameString\r
-                               if (CdmUtils.isBlank(commonNameString)){\r
+                               if (isBlank(commonNameString)){\r
                                        String message = "CommonName is empty or null. Do not import record for taxon " + taxonId;\r
                                        logger.warn(message);\r
                                        continue;\r
@@ -219,7 +204,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                \r
                                //taxon\r
                                Taxon taxon = null;\r
-                               TaxonBase taxonBase  = taxonMap.get(String.valueOf(taxonId));\r
+                               TaxonBase<?> taxonBase  = taxonMap.get(String.valueOf(taxonId));\r
                                if (taxonBase == null){\r
                                        logger.warn("Taxon (" + taxonId + ") could not be found. Common name " + commonNameString + "(" + commonNameId + ") not imported");\r
                                        continue;\r
@@ -247,7 +232,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                        NamedArea area = regionMap.get(regionFk);\r
                                        if (area == null){\r
                                                if (regionFkSplit.length > 1 && StringUtils.isNotBlank(regionFk)){\r
-                                                       logger.warn("Area for " + regionFk + " not defined.");\r
+                                                       logger.warn("Area for " + regionFk + " not defined in regionMap.");\r
                                                }else{\r
                                                        //no region is defined\r
                                                }\r
@@ -270,7 +255,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                        }\r
                                }\r
                                \r
-                               Reference<?> reference = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(languageRefRefFk));\r
+                               Reference<?> reference = refMap.get(String.valueOf(languageRefRefFk));\r
                                String microCitation = null;\r
                                String originalNameString = null;\r
                                \r
@@ -278,7 +263,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                if (nameInSourceFk != null && nameUsedInSource == null){\r
                                        logger.warn("Name used in source (" + nameInSourceFk + ") was not found for common name " + commonNameId);\r
                                }\r
-                               DescriptionElementSource source = DescriptionElementSource.NewInstance(reference, microCitation, nameUsedInSource, originalNameString);\r
+                               DescriptionElementSource source = DescriptionElementSource.NewPrimarySourceInstance(reference, microCitation, nameUsedInSource, originalNameString);\r
                                for (CommonTaxonName commonTaxonName : commonTaxonNames){\r
                                        commonTaxonName.addSource(source);\r
                                }\r
@@ -289,7 +274,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                        //Taxon misappliedName = getMisappliedName(biblioRefMap, nomRefMap, misNameRefFk, taxon);\r
                                        Taxon misappliedNameTaxon = null;\r
                                        if (misappliedTaxonId != null){\r
-                                               TaxonBase misTaxonBase =  taxonMap.get(String.valueOf(misappliedTaxonId));\r
+                                               TaxonBase<?> misTaxonBase =  taxonMap.get(String.valueOf(misappliedTaxonId));\r
                                                if (misTaxonBase == null){\r
                                                        logger.warn("MisappliedName not found for misappliedTaxonId " + misappliedTaxonId + "; commonNameId: " + commonNameId);\r
                                                }else if (misTaxonBase.isInstanceOf(Taxon.class)){\r
@@ -299,7 +284,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                                }\r
                                        }else{\r
                                                \r
-                                               Reference<?> sec = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(misNameRefFk));\r
+                                               Reference<?> sec = refMap.get(String.valueOf(misNameRefFk));\r
                                                if (nameUsedInSource == null || sec == null){\r
                                                        logger.warn("Taxon name or misapplied name reference is null for common name " + commonNameId);\r
                                                }else{\r
@@ -333,27 +318,26 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                \r
                                //reference extensions\r
                                if (reference != null){\r
-                                       if (CdmUtils.isNotEmpty(refLanguage)){\r
+                                       if (StringUtils.isNotBlank(refLanguage)){\r
                                                ExtensionType refLanguageExtensionType = getExtensionType( state, REFERENCE_LANGUAGE_STRING_UUID, "reference language","The language of the reference","ref. lang.");\r
                                                Extension.NewInstance(reference, refLanguage, refLanguageExtensionType);\r
                                        }\r
                                        \r
-                                       if (CdmUtils.isNotEmpty(refLanguageIso639_2)){\r
+                                       if (StringUtils.isNotBlank(refLanguageIso639_2)){\r
                                                ExtensionType refLanguageIsoExtensionType = getExtensionType( state, REFERENCE_LANGUAGE_ISO639_2_UUID, "reference language iso 639-2","The iso 639-2 code of the references language","ref. lang. 639-2");\r
                                                Extension.NewInstance(reference, refLanguageIso639_2, refLanguageIsoExtensionType);\r
                                        }\r
-                               }else if (CdmUtils.isNotEmpty(refLanguage) || CdmUtils.isNotEmpty(refLanguageIso639_2)){\r
+                               }else if (isNotBlank(refLanguage) || isNotBlank(refLanguageIso639_2)){\r
                                        logger.warn("Reference is null (" + languageRefRefFk + ") but refLanguage (" + CdmUtils.Nz(refLanguage) + ") or iso639_2 (" + CdmUtils.Nz(refLanguageIso639_2) + ") was not null for common name ("+ commonNameId +")");\r
                                }\r
                                \r
                                //status\r
-                               if (CdmUtils.isNotEmpty(status)){\r
+                               if (isNotBlank(status)){\r
                                        AnnotationType statusAnnotationType = getAnnotationType( state, STATUS_ANNOTATION_UUID, "status","The status of this object","status", null);\r
-                                       Annotation annotation = Annotation.NewInstance(status, statusAnnotationType, Language.DEFAULT());\r
                                        for (CommonTaxonName commonTaxonName : commonTaxonNames){\r
+                                               Annotation annotation = Annotation.NewInstance(status, statusAnnotationType, Language.DEFAULT());\r
                                                commonTaxonName.addAnnotation(annotation);\r
                                        }\r
-                                       \r
                                }\r
                                \r
                                //Notes\r
@@ -377,40 +361,6 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
 \r
        }\r
 \r
-\r
-\r
-       /**\r
-        * Not used anymore. Use MisappliedName RIdentifier instead\r
-        * @param biblioRefMap\r
-        * @param nomRefMap\r
-        * @param misNameRefFk\r
-        * @param taxon\r
-        */\r
-       private boolean isFirstMisappliedName = true;\r
-       private Taxon getMisappliedName(Map<String, Reference> biblioRefMap, Map<String, Reference> nomRefMap, Object misNameRefFk, Taxon taxon) {\r
-               Taxon misappliedTaxon = null;\r
-               Reference<?> misNameRef = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(misNameRefFk));\r
-               misappliedTaxon = Taxon.NewInstance(taxon.getName(), misNameRef);\r
-               Set<String> includeProperty = new HashSet<String>();\r
-               try {\r
-//                     //IMatchStrategy matchStrategy = DefaultMatchStrategy.NewInstance(TaxonBase.class);\r
-//                     //List<TaxonBase> misappliedList1 = getCommonService().findMatching(misappliedTaxon, matchStrategy);\r
-                       List<TaxonBase> misappliedList = getTaxonService().list(misappliedTaxon, includeProperty, null, null, null, null);\r
-                       if (misappliedList.size() > 0){\r
-                               misappliedTaxon = CdmBase.deproxy(misappliedList.get(0), Taxon.class);\r
-                       }\r
-               } catch (ClassCastException e) {\r
-                       logger.error(e.getMessage());\r
-                       if (isFirstMisappliedName){\r
-                               e.printStackTrace();\r
-                               isFirstMisappliedName = false;\r
-                       }\r
-               }\r
-               return misappliedTaxon;\r
-       }\r
-\r
-\r
-\r
        /**\r
         * @param iso6392Map\r
         * @param iso639_2\r
@@ -421,7 +371,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
         */\r
        private Language getAndHandleLanguage(Map<String, Language> iso639Map,  String iso639_2, String iso639_1, String languageString, String originalLanguageString, BerlinModelImportState state) {\r
                Language language;\r
-               if (CdmUtils.isNotEmpty(iso639_2)|| CdmUtils.isNotEmpty(iso639_1)  ){\r
+               if (isNotBlank(iso639_2)|| isNotBlank(iso639_1)  ){\r
                        //TODO test performance, implement in state\r
                        language = getLanguageFromIsoMap(iso639Map, iso639_2, iso639_1);\r
                        \r
@@ -429,7 +379,13 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                                language = getTermService().getLanguageByIso(iso639_2);\r
                                iso639Map.put(iso639_2, language);\r
                                if (language == null){\r
-                                       language = getTermService().getLanguageByIso(iso639_1);\r
+                                       try {\r
+                                               language = getTermService().getLanguageByIso(iso639_1);\r
+                                       } catch (Exception e) {\r
+                                               // TODO Auto-generated catch block\r
+                                               // TODO remove if problem with duplicate DescElement_Annot id is solved\r
+                                               e.printStackTrace();\r
+                                       }\r
                                        iso639Map.put(iso639_1, language);\r
                                }\r
                                if (language == null){\r
@@ -469,7 +425,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
         * @param originalLanguageString\r
         */\r
        private void addOriginalLanguage(Language language,     String originalLanguageString) {\r
-               if (CdmUtils.isBlank(originalLanguageString)){\r
+               if (isBlank(originalLanguageString)){\r
                        return;\r
                }else if (language == null){\r
                        logger.warn("Language could not be defined, but originalLanguageString exists: " + originalLanguageString);\r
@@ -503,7 +459,7 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                ResultSet rs = source.getResultSet(sql);\r
                while (rs.next()){\r
                        String strRegionFks = rs.getString("RegionFks"); \r
-                       if (StringUtils.isBlank(strRegionFks)){\r
+                       if (isBlank(strRegionFks)){\r
                                continue;\r
                        }\r
                        \r
@@ -576,16 +532,16 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                NamedArea area;\r
                if (tdwgCode.equalsIgnoreCase("Ab")){\r
                        area = getNamedArea(state, BerlinModelTransformer.uuidAzerbaijanNakhichevan, "Azerbaijan & Nakhichevan", "Azerbaijan (including Nakhichevan)",  "Ab", null, null);\r
-                       getTermService().save(area);\r
+                       getTermService().saveOrUpdate(area);\r
                }else if (tdwgCode.equalsIgnoreCase("Uk")){\r
                        area = getNamedArea(state, BerlinModelTransformer.uuidUkraineAndCrimea , "Ukraine & Crimea", "Ukraine (including Crimea)", "Uk", null, null);\r
-                       getTermService().save(area);\r
+                       getTermService().saveOrUpdate(area);\r
                }else if (tdwgCode.equalsIgnoreCase("Rf")){\r
-                       area = WaterbodyOrCountry.RUSSIANFEDERATION();\r
+                       area = Country.RUSSIANFEDERATION();\r
                }else if (tdwgCode.equalsIgnoreCase("Gg")){\r
-                       area = WaterbodyOrCountry.GEORGIA();\r
+                       area = Country.GEORGIA();\r
                }else{\r
-                       area = TdwgArea.getAreaByTdwgAbbreviation(tdwgCode);\r
+                       area = TdwgAreaProvider.getAreaByTdwgAbbreviation(tdwgCode);\r
                }\r
                if (area == null){\r
                        logger.warn("Area is null for " + tdwgCode);\r
@@ -606,8 +562,6 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                return sqlWhere;\r
        }\r
 \r
-\r
-\r
        /**\r
         * Returns a map which is filled by the emCode->TdwgCode mapping defined in emArea.\r
         * Some exceptions are defined for emCode 'Ab','Rf','Uk' and some additional mapping is added \r
@@ -645,8 +599,6 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
        }\r
 \r
 \r
-\r
-\r
        /**\r
         * Returns the first non-image gallery description. Creates a new one if no description exists.\r
         * @param taxon\r
@@ -665,49 +617,34 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                return result;\r
        }\r
 \r
-       /* (non-Javadoc)\r
-        * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)\r
-        */\r
+       @Override\r
        public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {\r
                String nameSpace;\r
                Class<?> cdmClass;\r
                Set<String> idSet;\r
                Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();\r
                \r
-               int pos = -1;\r
+               String pos = "0";\r
                try{\r
                        Set<String> taxonIdSet = new HashSet<String>();\r
                        Set<String> nameIdSet = new HashSet<String>();\r
                        Set<String> referenceIdSet = new HashSet<String>();\r
                        while (rs.next()){\r
-                               pos = 0;\r
                                handleForeignKey(rs, taxonIdSet, "taxonId");\r
-                               pos = 1;\r
                                handleForeignKey(rs, taxonIdSet, "misappliedTaxonId");\r
-                               pos = 2;\r
                                handleForeignKey(rs, referenceIdSet, "refId");\r
-                               pos = 3;\r
                                handleForeignKey(rs, referenceIdSet, "languageRefRefFk");\r
-                               pos = 4;\r
                                handleForeignKey(rs, nameIdSet, "NameInSourceFk");\r
-                               pos = 5;\r
                                handleForeignKey(rs, nameIdSet, "PTNameFk");\r
-                               pos = 6;\r
                                handleForeignKey(rs, referenceIdSet, "MisNameRefFk");\r
-                               pos = -2;\r
-                               \r
                        }\r
                        \r
-                       pos = 7;\r
-                       \r
                        //name map\r
                        nameSpace = BerlinModelTaxonNameImport.NAMESPACE;\r
                        cdmClass = TaxonNameBase.class;\r
                        idSet = nameIdSet;\r
                        Map<String, TaxonNameBase<?,?>> nameMap = (Map<String, TaxonNameBase<?,?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
                        result.put(nameSpace, nameMap);\r
-\r
-                       pos = 8;\r
                        \r
                        //taxon map\r
                        nameSpace = BerlinModelTaxonImport.NAMESPACE;\r
@@ -715,27 +652,14 @@ public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
                        idSet = taxonIdSet;\r
                        Map<String, TaxonBase<?>> taxonMap = (Map<String, TaxonBase<?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
                        result.put(nameSpace, taxonMap);\r
-\r
-                       pos = 9;\r
                        \r
-                       //nom reference map\r
-                       nameSpace = BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE;\r
+                       //reference map\r
+                       nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;\r
                        cdmClass = Reference.class;\r
                        idSet = referenceIdSet;\r
-                       Map<String, Reference> nomReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
-                       result.put(nameSpace, nomReferenceMap);\r
-\r
-                       pos = 10;\r
-                       \r
-                       //biblio reference map\r
-                       nameSpace = BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE;\r
-                       cdmClass = Reference.class;\r
-                       idSet = referenceIdSet;\r
-                       Map<String, Reference<?>> biblioReferenceMap = (Map<String, Reference<?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
-                       result.put(nameSpace, biblioReferenceMap);\r
-\r
-                       pos = 11;\r
-                       \r
+                       Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       result.put(nameSpace, referenceMap);\r
+                       // TODO remove if problem with duplicate DescElement_Annot id is solved\r
                } catch (SQLException e) {\r
                        throw new RuntimeException("pos: " +pos, e);\r
                } catch (NullPointerException nep){\r