merge pesi->trunk
[cdmlib.git] / cdmlib-io / src / main / java / eu / etaxonomy / cdm / io / berlinModel / in / BerlinModelFactsImport.java
index 63a5905adc2e95fe29e3755b95d8f5bbd6dcf011..6b669ff16a14ec4867fe71677019692204ef7bd0 100644 (file)
@@ -9,9 +9,14 @@
 \r
 package eu.etaxonomy.cdm.io.berlinModel.in;\r
 \r
+import java.io.IOException;\r
+import java.net.MalformedURLException;\r
+import java.net.URISyntaxException;\r
+import java.net.URL;\r
 import java.sql.ResultSet;\r
 import java.sql.SQLException;\r
 import java.util.Collection;\r
+import java.util.HashMap;\r
 import java.util.HashSet;\r
 import java.util.Map;\r
 import java.util.Set;\r
@@ -20,17 +25,27 @@ import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;\r
 \r
 import eu.etaxonomy.cdm.common.CdmUtils;\r
+import eu.etaxonomy.cdm.common.mediaMetaData.ImageMetaData;\r
 import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;\r
-import eu.etaxonomy.cdm.io.common.ICdmIO;\r
-import eu.etaxonomy.cdm.io.common.IImportConfigurator;\r
+import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelFactsImportValidator;\r
+import eu.etaxonomy.cdm.io.common.IOValidator;\r
 import eu.etaxonomy.cdm.io.common.MapWrapper;\r
+import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;\r
 import eu.etaxonomy.cdm.io.common.Source;\r
+import eu.etaxonomy.cdm.model.common.Annotation;\r
 import eu.etaxonomy.cdm.model.common.CdmBase;\r
+import eu.etaxonomy.cdm.model.common.DescriptionElementSource;\r
 import eu.etaxonomy.cdm.model.common.Language;\r
+import eu.etaxonomy.cdm.model.common.Marker;\r
+import eu.etaxonomy.cdm.model.common.MarkerType;\r
 import eu.etaxonomy.cdm.model.common.TermVocabulary;\r
+import eu.etaxonomy.cdm.model.description.DescriptionElementBase;\r
 import eu.etaxonomy.cdm.model.description.Feature;\r
 import eu.etaxonomy.cdm.model.description.TaxonDescription;\r
 import eu.etaxonomy.cdm.model.description.TextData;\r
+import eu.etaxonomy.cdm.model.media.ImageFile;\r
+import eu.etaxonomy.cdm.model.media.Media;\r
+import eu.etaxonomy.cdm.model.media.MediaRepresentation;\r
 import eu.etaxonomy.cdm.model.reference.ReferenceBase;\r
 import eu.etaxonomy.cdm.model.taxon.Taxon;\r
 import eu.etaxonomy.cdm.model.taxon.TaxonBase;\r
@@ -45,24 +60,21 @@ import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
 public class BerlinModelFactsImport  extends BerlinModelImportBase {\r
        private static final Logger logger = Logger.getLogger(BerlinModelFactsImport.class);\r
 \r
+       public static final String NAMESPACE = "Fact";\r
+       \r
+       public static final String SEQUENCE_PREFIX = "ORDER: ";\r
+       \r
        private int modCount = 10000;\r
+       private static final String pluralString = "facts";\r
+       private static final String dbTableName = "Fact";\r
+\r
+       //FIXME don't use as class variable\r
+       private MapWrapper<Feature> featureMap;\r
        \r
        public BerlinModelFactsImport(){\r
                super();\r
        }\r
 \r
-       /* (non-Javadoc)\r
-        * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)\r
-        */\r
-       @Override\r
-       protected boolean doCheck(IImportConfigurator config){\r
-               boolean result = true;\r
-               logger.warn("Checking for Facts not yet implemented");\r
-               //result &= checkArticlesWithoutJournal(bmiConfig);\r
-               //result &= checkPartOfJournal(bmiConfig);\r
-               \r
-               return result;\r
-       }\r
 \r
        private TermVocabulary<Feature> getFeatureVocabulary(){\r
                try {\r
@@ -78,11 +90,8 @@ public class BerlinModelFactsImport  extends BerlinModelImportBase {
        \r
        private MapWrapper<Feature> invokeFactCategories(BerlinModelImportConfigurator bmiConfig){\r
                \r
-//             Map<Integer, Feature> featureMap = new HashMap<Integer, Feature>();\r
                MapWrapper<Feature> result = bmiConfig.getFeatureMap();\r
-\r
                Source source = bmiConfig.getSource();\r
-\r
                \r
                try {\r
                        //get data from database\r
@@ -120,12 +129,10 @@ public class BerlinModelFactsImport  extends BerlinModelImportBase {
 //                                     RankRestrictionFk       int     Checked\r
                                }\r
                                                                \r
-                       //      featureMap.put(factCategoryId, feature);\r
                                result.put(factCategoryId, feature);\r
-       \r
                        }\r
                        Collection<Feature> col = result.getAllValues();\r
-                       getTermService().saveTermsAll(col);\r
+                       getTermService().save((Collection)col);\r
                        return result;\r
                } catch (SQLException e) {\r
                        logger.error("SQLException:" +  e);\r
@@ -133,103 +140,140 @@ public class BerlinModelFactsImport  extends BerlinModelImportBase {
                }\r
 \r
        }\r
+\r
+       /* (non-Javadoc)\r
+        * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#doInvoke(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)\r
+        */\r
+       @Override\r
+       protected boolean doInvoke(BerlinModelImportState state) {\r
+               featureMap = invokeFactCategories(state.getConfig());\r
+               return super.doInvoke(state);\r
+       }\r
        \r
 \r
        /* (non-Javadoc)\r
-        * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IImportConfigurator, eu.etaxonomy.cdm.api.application.CdmApplicationController, java.util.Map)\r
+        * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)\r
         */\r
        @Override\r
-       protected boolean doInvoke(BerlinModelImportState<BerlinModelImportConfigurator> state){\r
-               boolean result = true;\r
-               \r
-               MapWrapper<TaxonBase> taxonMap = (MapWrapper<TaxonBase>)state.getStore(ICdmIO.TAXON_STORE);\r
-               MapWrapper<ReferenceBase> referenceMap = (MapWrapper<ReferenceBase>)state.getStore(ICdmIO.REFERENCE_STORE);\r
-               MapWrapper<ReferenceBase> nomRefMap = (MapWrapper<ReferenceBase>)state.getStore(ICdmIO.NOMREF_STORE);\r
-               \r
-               Set<TaxonBase> taxonStore = new HashSet<TaxonBase>();\r
-               \r
+       protected String getRecordQuery(BerlinModelImportConfigurator config) {\r
+               String strQuery = \r
+                       " SELECT Fact.*, PTaxon.RIdentifier as taxonId, RefDetail.Details " + \r
+                       " FROM Fact " +\r
+               " INNER JOIN PTaxon ON Fact.PTNameFk = PTaxon.PTNameFk AND Fact.PTRefFk = PTaxon.PTRefFk " +\r
+               " LEFT OUTER JOIN RefDetail ON Fact.FactRefDetailFk = RefDetail.RefDetailId AND Fact.FactRefFk = RefDetail.RefFk " +\r
+               " WHERE (FactId IN (" + ID_LIST_TOKEN + "))" + \r
+                " ORDER By Sequence";\r
+               return strQuery;\r
+       }\r
+       \r
+\r
+       /* (non-Javadoc)\r
+        * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)\r
+        */\r
+       public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {\r
+               boolean success = true ;\r
                BerlinModelImportConfigurator config = state.getConfig();\r
-               Source source = config.getSource();\r
-               \r
-               logger.info("start makeFacts ...");\r
-               \r
-               MapWrapper<Feature> featureMap = invokeFactCategories(config);\r
-               \r
-               //for testing only\r
-               //TaxonBase taxonBase = Taxon.NewInstance(BotanicalName.NewInstance(null), null);\r
+               Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();\r
+               Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);\r
+               Map<String, ReferenceBase> biblioRefMap = (Map<String, ReferenceBase>) partitioner.getObjectMap(BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE);\r
+               Map<String, ReferenceBase> nomRefMap = (Map<String, ReferenceBase>) partitioner.getObjectMap(BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE);\r
+\r
+               ResultSet rs = partitioner.getResultSet();\r
                \r
+               ReferenceBase<?> sourceRef = state.getConfig().getSourceReference();\r
                \r
-               try {\r
-                       //get data from database\r
-                       String strQuery = \r
-                                       " SELECT Fact.*, PTaxon.RIdentifier as taxonId, RefDetail.Details " + \r
-                                       " FROM Fact " +\r
-                       " INNER JOIN PTaxon ON Fact.PTNameFk = PTaxon.PTNameFk AND Fact.PTRefFk = PTaxon.PTRefFk " +\r
-                       " LEFT OUTER JOIN RefDetail ON Fact.FactRefDetailFk = RefDetail.RefDetailId AND Fact.FactRefFk = RefDetail.RefFk " +\r
-                       " WHERE (1=1)";\r
-                       ResultSet rs = source.getResultSet(strQuery) ;\r
-\r
+               try{\r
                        int i = 0;\r
-                       //for each reference\r
+                       //for each fact\r
                        while (rs.next()){\r
                                try{\r
                                        if ((i++ % modCount) == 0){ logger.info("Facts handled: " + (i-1));}\r
                                        \r
-                                       //Map<String, Object> valueMap = getValueMap(rs);\r
-                                       \r
                                        int factId = rs.getInt("factId");\r
-                                       \r
-                                       \r
                                        Object taxonIdObj = rs.getObject("taxonId");\r
                                        int taxonId = rs.getInt("taxonId");\r
                                        Object factRefFkObj = rs.getObject("factRefFk");\r
-                                       int factRefFk = rs.getInt("factRefFk");\r
                                        Object categoryFkObj = rs.getObject("factCategoryFk");\r
                                        Integer categoryFk = rs.getInt("factCategoryFk");\r
-                                       \r
                                        String details = rs.getString("Details");\r
-       //                              int ptDesignationRefFk = rs.getInt("PTDesignationRefFk");\r
-       //                              String ptDesignation details = rs.getInt("PTDesignationRefDetailFk");\r
                                        String fact = CdmUtils.Nz(rs.getString("Fact"));\r
                                        String notes = CdmUtils.Nz(rs.getString("notes"));\r
+                                       Boolean doubtfulFlag = rs.getBoolean("DoubtfulFlag");\r
+                                       Boolean publishFlag = rs.getBoolean("publishFlag");\r
                                        \r
+                                       TaxonBase taxonBase = getTaxon(taxonMap, taxonIdObj, taxonId);\r
+                                       Feature feature = getFeature(featureMap, categoryFkObj, categoryFk) ;\r
                                        \r
-                                       \r
-                                       TaxonBase taxonBase;\r
-                                       if (taxonIdObj != null){\r
-                                               taxonBase = taxonMap.get(taxonId);\r
-                                       }else{\r
-                                               taxonBase = null;\r
-                                       }\r
-                                       Feature feature;\r
-                                       if (categoryFkObj != null){\r
-                                               feature = featureMap.get(categoryFk); \r
+                                       if (taxonBase == null){\r
+                                               logger.warn("Taxon for Fact " + factId + " does not exist in store");\r
+                                               success = false;\r
                                        }else{\r
-                                               feature = null;\r
-                                       }\r
-                                       \r
-                                       if (taxonBase != null){\r
                                                Taxon taxon;\r
                                                if ( taxonBase instanceof Taxon ) {\r
                                                        taxon = (Taxon) taxonBase;\r
                                                }else{\r
                                                        logger.warn("TaxonBase " + (taxonIdObj==null?"(null)":taxonIdObj) + " for Fact " + factId + " was not of type Taxon but: " + taxonBase.getClass().getSimpleName());\r
+                                                       success = false;\r
                                                        continue;\r
                                                }\r
                                                \r
-                                               TaxonDescription taxonDescription;\r
+                                               TaxonDescription taxonDescription = null;\r
                                                Set<TaxonDescription> descriptionSet= taxon.getDescriptions();\r
-                                               if (descriptionSet.size() > 0) {\r
-                                                       taxonDescription = descriptionSet.iterator().next(); \r
-                                               }else{\r
-                                                       taxonDescription = TaxonDescription.NewInstance();\r
-                                                       taxon.addDescription(taxonDescription);\r
-                                               }\r
                                                \r
+                                               boolean isImage = false;\r
+                                               Media media = null;\r
+                                               //for diptera images\r
+                                               if (categoryFk == 51){  //TODO check also FactCategory string\r
+                                                       isImage = true;\r
+                                                       media = Media.NewInstance();\r
+                                                       taxonDescription = makeImage(state, fact, media, descriptionSet, taxon);\r
+                                                       if (taxonDescription == null){\r
+                                                               continue;\r
+                                                       }\r
+                                               }\r
+                                               //all others (no image)\r
+                                               else{ \r
+                                                       for (TaxonDescription desc: descriptionSet){\r
+                                                               if (! desc.isImageGallery()){\r
+                                                                       taxonDescription = desc;\r
+                                                               }\r
+                                                       }\r
+                                                       if (taxonDescription == null){\r
+                                                               taxonDescription = TaxonDescription.NewInstance();\r
+                                                               taxonDescription.setTitleCache(sourceRef == null ? null : sourceRef.getTitleCache());\r
+                                                               taxon.addDescription(taxonDescription);\r
+                                                       }\r
+                                               }\r
+                                       \r
                                                //textData\r
-                                               TextData textData = TextData.NewInstance();\r
-                                               //TODO textData.putText(fact, bmiConfig.getFactLanguage());  //doesn't work because  bmiConfig.getFactLanguage() is not not a persistent Language Object\r
-                                               //throws  in thread "main" org.springframework.dao.InvalidDataAccessApiUsageException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language; nested exception is org.hibernate.TransientObjectException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language\r
+                                               TextData textData = null;\r
+                                               boolean newTextData = true;\r
+       \r
+                                               // For Cichorieae DB: If fact category is 31 (Systematics) and there is already a Systematics TextData \r
+                                               // description element append the fact text to the existing TextData\r
+                                               if(categoryFk == 31) {\r
+                                                       Set<DescriptionElementBase> descriptionElements = taxonDescription.getElements();\r
+                                                       for (DescriptionElementBase descriptionElement : descriptionElements) {\r
+                                                               String featureString = descriptionElement.getFeature().getRepresentation(Language.DEFAULT()).getLabel();\r
+                                                               if (descriptionElement instanceof TextData && featureString.equals("Systematics")) { // TODO: test\r
+                                                                       textData = (TextData)descriptionElement;\r
+                                                                       String factTextStr = textData.getText(Language.DEFAULT());\r
+                                                                       // FIXME: Removing newlines doesn't work\r
+                                                                       if (factTextStr.contains("\\r\\n")) {\r
+                                                                               factTextStr = factTextStr.replaceAll("\\r\\n","");\r
+                                                                       }\r
+                                                                       StringBuilder factText = new StringBuilder(factTextStr);\r
+                                                                       factText.append(fact);\r
+                                                                       fact = factText.toString();\r
+                                                                       newTextData = false;\r
+                                                                       break;\r
+                                                               }\r
+                                                       }\r
+                                               }\r
+                                               \r
+                                               if(newTextData == true) { \r
+                                                       textData = TextData.NewInstance(); \r
+                                               }\r
                                                \r
                                                //for diptera database\r
                                                if (categoryFk == 99 && notes.contains("<OriginalName>")){\r
@@ -237,78 +281,241 @@ public class BerlinModelFactsImport  extends BerlinModelImportBase {
                                                        notes = notes.replaceAll("</OriginalName>", "");\r
                                                        fact = notes + ": " +  fact ;\r
                                                }\r
-                                               textData.putText(fact, Language.DEFAULT());\r
-                                               textData.setType(feature);\r
-                                               \r
-                                               //\r
-                                               ReferenceBase citation;\r
-                                               if (factRefFkObj != null){\r
-                                                       citation = referenceMap.get(factRefFk); \r
-                                                       if (citation == null){\r
-                                                               citation = nomRefMap.get(factRefFk);\r
-                                                       }\r
-                                                       if (citation == null && (factRefFk != 0)){\r
-                                                               logger.warn("Citation not found in referenceMap: " + factRefFk);\r
-                                                       }\r
+                                               //TODO textData.putText(fact, bmiConfig.getFactLanguage());  //doesn't work because  bmiConfig.getFactLanguage() is not not a persistent Language Object\r
+                                               //throws  in thread "main" org.springframework.dao.InvalidDataAccessApiUsageException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language; nested exception is org.hibernate.TransientObjectException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language\r
+                                               if (isImage){\r
+                                                       textData.addMedia(media);\r
+                                                       textData.setType(Feature.IMAGE());\r
                                                }else{\r
-                                                       citation = null;\r
+                                                       textData.putText(fact, Language.DEFAULT());\r
+                                                       textData.setType(feature);\r
                                                }\r
-\r
                                                \r
-                                               textData.setCitation(citation);\r
-                                               textData.setCitationMicroReference(details);\r
+                                               //reference\r
+                                               ReferenceBase citation = null;\r
+                                               String factRefFk = String.valueOf(factRefFkObj);\r
+                                               if (factRefFkObj != null){\r
+                                                       citation = getReferenceOnlyFromMaps(\r
+                                                                       biblioRefMap, nomRefMap, factRefFk);    \r
+                                               }\r
+                                               if (citation == null && (factRefFkObj != null)){\r
+                                                       logger.warn("Citation not found in referenceMap: " + factRefFk);\r
+                                                       success = false;\r
+                                               }\r
+                                               if (citation != null || CdmUtils.isNotEmpty(details)){\r
+                                                       DescriptionElementSource originalSource = DescriptionElementSource.NewInstance();\r
+                                                       originalSource.setCitation(citation);\r
+                                                       originalSource.setCitationMicroReference(details);\r
+                                                       textData.addSource(originalSource);\r
+                                               }\r
                                                taxonDescription.addElement(textData);\r
+                                               //doubtfulFlag\r
+                                               if (doubtfulFlag){\r
+                                                       textData.addMarker(Marker.NewInstance(MarkerType.IS_DOUBTFUL(), true));\r
+                                               }\r
+                                               //publisheFlag\r
+                                               textData.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), publishFlag));\r
+                                               //Sequence\r
+                                               Integer sequence = rs.getInt("Sequence");\r
+                                               if (sequence != null && sequence != 999){\r
+                                                       String strSequence = String.valueOf(sequence);\r
+                                                       strSequence = SEQUENCE_PREFIX + strSequence;\r
+                                                       //TODO make it an Extension when possible\r
+                                                       //Extension datesExtension = Extension.NewInstance(textData, strSequence, ExtensionType.ORDER());\r
+                                                       Annotation annotation = Annotation.NewInstance(strSequence, Language.DEFAULT());\r
+                                                       textData.addAnnotation(annotation);\r
+                                               }\r
                                                \r
+                                               //                                              if (categoryFkObj == FACT_DESCRIPTION){\r
+       //                                              //;\r
+       //                                      }else if (categoryFkObj == FACT_OBSERVATION){\r
+       //                                              //;\r
+       //                                      }else if (categoryFkObj == FACT_DISTRIBUTION_EM){\r
+       //                                              //\r
+       //                                      }else {\r
+       //                                              //TODO\r
+       //                                              //logger.warn("FactCategory " + categoryFk + " not yet implemented");\r
+       //                                      }\r
                                                \r
-//                                             if (categoryFkObj == FACT_DESCRIPTION){\r
-//                                                     //;\r
-//                                             }else if (categoryFkObj == FACT_OBSERVATION){\r
-//                                                     //;\r
-//                                             }else if (categoryFkObj == FACT_DISTRIBUTION_EM){\r
-//                                                     //\r
-//                                             }else {\r
-//                                                     //TODO\r
-//                                                     //logger.warn("FactCategory " + categoryFk + " not yet implemented");\r
-//                                             }\r
+                                               //notes\r
+                                               doCreatedUpdatedNotes(state, textData, rs);\r
                                                \r
                                                //TODO\r
-                                               //References\r
-                                               //factId\r
+                                               //Designation References -> unclear how to map to CDM\r
+                                               //factId -> OriginalSource for descriptionElements not yet implemented\r
                                                \r
-                                               //etc.\r
-                                               doCreatedUpdatedNotes(config, textData, rs, "Fact");\r
-\r
+                                               //sequence -> textData is not an identifiable entity therefore extensions are not possible\r
+                                               //fact category better\r
                                                \r
-                                               taxonStore.add(taxon);\r
-                                       }else{\r
-                                               //TODO\r
-                                               logger.warn("Taxon for Fact " + factId + " does not exist in store");\r
+                                               taxaToSave.add(taxon);\r
                                        }\r
-                               } catch (RuntimeException re){\r
-                                       logger.error("A runtime exception occurred during the facts import");\r
-                                       result = false;\r
-                                       throw re;\r
+                               } catch (Exception re){\r
+                                       logger.error("An exception occurred during the facts import");\r
+                                       re.printStackTrace();\r
+                                       success = false;\r
                                }\r
                                //put\r
                        }\r
                        logger.info("Facts handled: " + (i-1));\r
-                       logger.info("Taxa to save: " + taxonStore.size());\r
-                       getTaxonService().saveTaxonAll(taxonStore);     \r
+                       logger.info("Taxa to save: " + taxaToSave.size());\r
+                       getTaxonService().save(taxaToSave);     \r
+               }catch(SQLException e){\r
+                       throw new RuntimeException(e);\r
+               }\r
+               return success;\r
+       }\r
+\r
+       /* (non-Javadoc)\r
+        * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)\r
+        */\r
+       public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {\r
+               String nameSpace;\r
+               Class cdmClass;\r
+               Set<String> idSet;\r
+               Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();\r
+               \r
+               try{\r
+                       Set<String> taxonIdSet = new HashSet<String>();\r
+                       Set<String> referenceIdSet = new HashSet<String>();\r
+                       Set<String> refDetailIdSet = new HashSet<String>();\r
+                       while (rs.next()){\r
+                               handleForeignKey(rs, taxonIdSet, "taxonId");\r
+                               handleForeignKey(rs, referenceIdSet, "FactRefFk");\r
+                               handleForeignKey(rs, referenceIdSet, "PTDesignationRefFk");\r
+                               handleForeignKey(rs, refDetailIdSet, "FactRefDetailFk");\r
+                               handleForeignKey(rs, refDetailIdSet, "PTDesignationRefDetailFk");\r
+                       }\r
                        \r
-                       logger.info("end makeFacts ...");\r
-                       return result;\r
+                       //taxon map\r
+                       nameSpace = BerlinModelTaxonImport.NAMESPACE;\r
+                       cdmClass = TaxonBase.class;\r
+                       idSet = taxonIdSet;\r
+                       Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       result.put(nameSpace, taxonMap);\r
+\r
+\r
+                       //nom reference map\r
+                       nameSpace = BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE;\r
+                       cdmClass = ReferenceBase.class;\r
+                       idSet = referenceIdSet;\r
+                       Map<String, ReferenceBase> nomReferenceMap = (Map<String, ReferenceBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       result.put(nameSpace, nomReferenceMap);\r
+\r
+                       //biblio reference map\r
+                       nameSpace = BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE;\r
+                       cdmClass = ReferenceBase.class;\r
+                       idSet = referenceIdSet;\r
+                       Map<String, ReferenceBase> biblioReferenceMap = (Map<String, ReferenceBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       result.put(nameSpace, biblioReferenceMap);\r
+                       \r
+                       //nom refDetail map\r
+                       nameSpace = BerlinModelRefDetailImport.NOM_REFDETAIL_NAMESPACE;\r
+                       cdmClass = ReferenceBase.class;\r
+                       idSet = refDetailIdSet;\r
+                       Map<String, ReferenceBase> nomRefDetailMap= (Map<String, ReferenceBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       result.put(nameSpace, nomRefDetailMap);\r
+                       \r
+                       //biblio refDetail map\r
+                       nameSpace = BerlinModelRefDetailImport.BIBLIO_REFDETAIL_NAMESPACE;\r
+                       cdmClass = ReferenceBase.class;\r
+                       idSet = refDetailIdSet;\r
+                       Map<String, ReferenceBase> biblioRefDetailMap= (Map<String, ReferenceBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);\r
+                       result.put(nameSpace, biblioRefDetailMap);\r
+       \r
                } catch (SQLException e) {\r
-                       logger.error("SQLException:" +  e);\r
-                       return false;\r
+                       throw new RuntimeException(e);\r
+               }\r
+               return result;\r
+       }\r
+       \r
+       \r
+       /**\r
+        * @param state \r
+        * @param media \r
+        * @param media \r
+        * @param descriptionSet \r
+        * \r
+        */\r
+       private TaxonDescription makeImage(BerlinModelImportState state, String fact, Media media, Set<TaxonDescription> descriptionSet, Taxon taxon) {\r
+               TaxonDescription taxonDescription = null;\r
+               ReferenceBase sourceRef = state.getConfig().getSourceReference();\r
+               String uri = fact;\r
+               Integer size = null; \r
+               ImageMetaData imageMetaData = ImageMetaData.newInstance();\r
+               URL url;\r
+               try {\r
+                       url = new URL(fact.trim());\r
+               } catch (MalformedURLException e) {\r
+                       logger.warn("Malformed URL. Image could not be imported: " + CdmUtils.Nz(uri));\r
+                       return null;\r
+               }\r
+               try {\r
+                       imageMetaData.readMetaData(url.toURI(), 0);\r
                }\r
+               catch(URISyntaxException e){\r
+                       e.printStackTrace();\r
+               }\r
+               \r
+               MediaRepresentation mediaRepresentation = MediaRepresentation.NewInstance(imageMetaData.getMimeType(), null);\r
+               media.addRepresentation(mediaRepresentation);\r
+               ImageFile image = ImageFile.NewInstance(uri, size, imageMetaData);\r
+               mediaRepresentation.addRepresentationPart(image);\r
+               \r
+               taxonDescription = taxon.getOrCreateImageGallery(sourceRef == null ? null :sourceRef.getTitleCache());\r
+               \r
+               return taxonDescription;\r
+       }\r
 \r
+       private TaxonBase getTaxon(Map<String, TaxonBase> taxonMap, Object taxonIdObj, Integer taxonId){\r
+               if (taxonIdObj != null){\r
+                       return taxonMap.get(String.valueOf(taxonId));\r
+               }else{\r
+                       return null;\r
+               }\r
+               \r
+       }\r
+       \r
+       private Feature getFeature(MapWrapper<Feature> featureMap, Object categoryFkObj, Integer categoryFk){\r
+               if (categoryFkObj != null){\r
+                       return featureMap.get(categoryFk); \r
+               }else{\r
+                       return null;\r
+               }\r
+               \r
+       }\r
+       \r
+\r
+       /* (non-Javadoc)\r
+        * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)\r
+        */\r
+       @Override\r
+       protected boolean doCheck(BerlinModelImportState state){\r
+               IOValidator<BerlinModelImportState> validator = new BerlinModelFactsImportValidator();\r
+               return validator.validate(state);\r
+       }\r
+       \r
+       /* (non-Javadoc)\r
+        * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName()\r
+        */\r
+       @Override\r
+       protected String getTableName() {\r
+               return dbTableName;\r
+       }\r
+       \r
+       /* (non-Javadoc)\r
+        * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString()\r
+        */\r
+       @Override\r
+       public String getPluralString() {\r
+               return pluralString;\r
        }\r
        \r
        /* (non-Javadoc)\r
         * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)\r
         */\r
-       protected boolean isIgnore(IImportConfigurator config){\r
-               return ! config.isDoFacts();\r
+       protected boolean isIgnore(BerlinModelImportState state){\r
+               return ! state.getConfig().isDoFacts();\r
        }\r
 \r
+\r
 }\r