2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.berlinModel
.in
;
12 import java
.net
.MalformedURLException
;
14 import java
.sql
.ResultSet
;
15 import java
.sql
.SQLException
;
16 import java
.util
.Collection
;
17 import java
.util
.HashSet
;
20 import org
.apache
.log4j
.Logger
;
21 import org
.springframework
.stereotype
.Component
;
23 import eu
.etaxonomy
.cdm
.common
.CdmUtils
;
24 import eu
.etaxonomy
.cdm
.common
.MediaMetaData
.ImageMetaData
;
25 import eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
;
26 import eu
.etaxonomy
.cdm
.io
.common
.ICdmIO
;
27 import eu
.etaxonomy
.cdm
.io
.common
.MapWrapper
;
28 import eu
.etaxonomy
.cdm
.io
.common
.Source
;
29 import eu
.etaxonomy
.cdm
.model
.common
.Annotation
;
30 import eu
.etaxonomy
.cdm
.model
.common
.DescriptionElementSource
;
31 import eu
.etaxonomy
.cdm
.model
.common
.Language
;
32 import eu
.etaxonomy
.cdm
.model
.common
.Marker
;
33 import eu
.etaxonomy
.cdm
.model
.common
.MarkerType
;
34 import eu
.etaxonomy
.cdm
.model
.common
.TermVocabulary
;
35 import eu
.etaxonomy
.cdm
.model
.description
.DescriptionElementBase
;
36 import eu
.etaxonomy
.cdm
.model
.description
.Feature
;
37 import eu
.etaxonomy
.cdm
.model
.description
.TaxonDescription
;
38 import eu
.etaxonomy
.cdm
.model
.description
.TextData
;
39 import eu
.etaxonomy
.cdm
.model
.media
.ImageFile
;
40 import eu
.etaxonomy
.cdm
.model
.media
.Media
;
41 import eu
.etaxonomy
.cdm
.model
.media
.MediaRepresentation
;
42 import eu
.etaxonomy
.cdm
.model
.reference
.ReferenceBase
;
43 import eu
.etaxonomy
.cdm
.model
.taxon
.Taxon
;
44 import eu
.etaxonomy
.cdm
.model
.taxon
.TaxonBase
;
45 import eu
.etaxonomy
.cdm
.strategy
.exceptions
.UnknownCdmTypeException
;
53 public class BerlinModelFactsImport
extends BerlinModelImportBase
{
54 private static final Logger logger
= Logger
.getLogger(BerlinModelFactsImport
.class);
56 public static final String SEQUENCE_PREFIX
= "ORDER: ";
58 private int modCount
= 10000;
60 public BerlinModelFactsImport(){
65 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)
68 protected boolean doCheck(BerlinModelImportState state
){
69 boolean result
= true;
70 BerlinModelImportConfigurator bmiConfig
= state
.getConfig();
71 logger
.warn("Checking for Facts not yet fully implemented");
72 result
&= checkDesignationRefsExist(bmiConfig
);
76 private TermVocabulary
<Feature
> getFeatureVocabulary(){
78 //TODO work around until service method works
79 TermVocabulary
<Feature
> featureVocabulary
= BerlinModelTransformer
.factCategory2Feature(1).getVocabulary();
80 //TermVocabulary<Feature> vocabulary = getTermService().getVocabulary(vocabularyUuid);
81 return featureVocabulary
;
82 } catch (UnknownCdmTypeException e
) {
83 logger
.error("Feature vocabulary not available. New vocabulary created");
84 return new TermVocabulary
<Feature
>() ;
88 private MapWrapper
<Feature
> invokeFactCategories(BerlinModelImportConfigurator bmiConfig
){
90 MapWrapper
<Feature
> result
= bmiConfig
.getFeatureMap();
91 Source source
= bmiConfig
.getSource();
94 //get data from database
96 " SELECT FactCategory.* " +
97 " FROM FactCategory "+
99 ResultSet rs
= source
.getResultSet(strQuery
) ;
102 TermVocabulary
<Feature
> featureVocabulary
= getFeatureVocabulary();
107 if ((i
++ % modCount
) == 0 && i
!= 1 ){ logger
.info("FactCategories handled: " + (i
-1));}
109 int factCategoryId
= rs
.getInt("factCategoryId");
110 String factCategory
= rs
.getString("factCategory");
115 feature
= BerlinModelTransformer
.factCategory2Feature(factCategoryId
);
116 } catch (UnknownCdmTypeException e
) {
117 logger
.warn("New Feature (FactCategoryId: " + factCategoryId
+ ")");
118 feature
= Feature
.NewInstance(factCategory
, factCategory
, null);
119 feature
.setVocabulary(featureVocabulary
);
120 feature
.setSupportsTextData(true);
122 // MaxFactNumber int Checked
123 // ExtensionTableName varchar(100) Checked
124 // Description nvarchar(1000) Checked
125 // locExtensionFormName nvarchar(80) Checked
126 // RankRestrictionFk int Checked
129 // featureMap.put(factCategoryId, feature);
130 result
.put(factCategoryId
, feature
);
133 Collection
<Feature
> col
= result
.getAllValues();
134 getTermService().saveTermsAll(col
);
136 } catch (SQLException e
) {
137 logger
.error("SQLException:" + e
);
145 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IImportConfigurator, eu.etaxonomy.cdm.api.application.CdmApplicationController, java.util.Map)
148 protected boolean doInvoke(BerlinModelImportState state
) {
149 boolean result
= true;
151 MapWrapper
<TaxonBase
> taxonMap
= (MapWrapper
<TaxonBase
>)state
.getStore(ICdmIO
.TAXON_STORE
);
152 MapWrapper
<ReferenceBase
> referenceMap
= (MapWrapper
<ReferenceBase
>)state
.getStore(ICdmIO
.REFERENCE_STORE
);
153 MapWrapper
<ReferenceBase
> nomRefMap
= (MapWrapper
<ReferenceBase
>)state
.getStore(ICdmIO
.NOMREF_STORE
);
155 Set
<TaxonBase
> taxonStore
= new HashSet
<TaxonBase
>();
157 BerlinModelImportConfigurator config
= state
.getConfig();
158 Source source
= config
.getSource();
160 logger
.info("start makeFacts ...");
162 MapWrapper
<Feature
> featureMap
= invokeFactCategories(config
);
165 //get data from database
167 " SELECT Fact.*, PTaxon.RIdentifier as taxonId, RefDetail.Details " +
169 " INNER JOIN PTaxon ON Fact.PTNameFk = PTaxon.PTNameFk AND Fact.PTRefFk = PTaxon.PTRefFk " +
170 " LEFT OUTER JOIN RefDetail ON Fact.FactRefDetailFk = RefDetail.RefDetailId AND Fact.FactRefFk = RefDetail.RefFk " +
172 " ORDER By Sequence";
173 ResultSet rs
= source
.getResultSet(strQuery
) ;
174 ReferenceBase
<?
> sourceRef
= state
.getConfig().getSourceReference();
180 if ((i
++ % modCount
) == 0){ logger
.info("Facts handled: " + (i
-1));}
182 int factId
= rs
.getInt("factId");
183 Object taxonIdObj
= rs
.getObject("taxonId");
184 int taxonId
= rs
.getInt("taxonId");
185 Object factRefFkObj
= rs
.getObject("factRefFk");
186 int factRefFk
= rs
.getInt("factRefFk");
187 Object categoryFkObj
= rs
.getObject("factCategoryFk");
188 Integer categoryFk
= rs
.getInt("factCategoryFk");
189 String details
= rs
.getString("Details");
190 String fact
= CdmUtils
.Nz(rs
.getString("Fact"));
191 String notes
= CdmUtils
.Nz(rs
.getString("notes"));
192 Boolean doubtfulFlag
= rs
.getBoolean("DoubtfulFlag");
193 Boolean publishFlag
= rs
.getBoolean("publishFlag");
195 TaxonBase taxonBase
= getTaxon(taxonMap
, taxonIdObj
, taxonId
);
196 Feature feature
= getFeature(featureMap
, categoryFkObj
, categoryFk
) ;
198 if (taxonBase
== null){
199 logger
.warn("Taxon for Fact " + factId
+ " does not exist in store");
203 if ( taxonBase
instanceof Taxon
) {
204 taxon
= (Taxon
) taxonBase
;
206 logger
.warn("TaxonBase " + (taxonIdObj
==null?
"(null)":taxonIdObj
) + " for Fact " + factId
+ " was not of type Taxon but: " + taxonBase
.getClass().getSimpleName());
211 TaxonDescription taxonDescription
= null;
212 Set
<TaxonDescription
> descriptionSet
= taxon
.getDescriptions();
214 boolean isImage
= false;
217 if (categoryFk
== 51){ //TODO check also FactCategory string
219 media
= Media
.NewInstance();
220 taxonDescription
= makeImage(state
, fact
, media
, descriptionSet
, taxon
);
221 if (taxonDescription
== null){
225 //all others (no image)
227 for (TaxonDescription desc
: descriptionSet
){
228 if (! desc
.isImageGallery()){
229 taxonDescription
= desc
;
232 if (taxonDescription
== null){
233 taxonDescription
= TaxonDescription
.NewInstance();
234 taxonDescription
.setTitleCache(sourceRef
== null ?
null:sourceRef
.getTitleCache());
235 taxon
.addDescription(taxonDescription
);
240 TextData textData
= null;
241 boolean newTextData
= true;
243 // For Cichorieae DB: If fact category is 31 (Systematics) and there is already a Systematics TextData
244 // description element append the fact text to the existing TextData
245 if(categoryFk
== 31) {
246 Set
<DescriptionElementBase
> descriptionElements
= taxonDescription
.getElements();
247 for (DescriptionElementBase descriptionElement
: descriptionElements
) {
248 String featureString
= descriptionElement
.getFeature().getRepresentation(Language
.DEFAULT()).getLabel();
249 if (descriptionElement
instanceof TextData
&& featureString
.equals("Systematics")) { // TODO: test
250 textData
= (TextData
)descriptionElement
;
251 String factTextStr
= textData
.getText(Language
.DEFAULT());
252 // FIXME: Removing newlines doesn't work
253 if (factTextStr
.contains("\\r\\n")) {
254 factTextStr
= factTextStr
.replaceAll("\\r\\n","");
256 StringBuilder factText
= new StringBuilder(factTextStr
);
257 factText
.append(fact
);
258 fact
= factText
.toString();
265 if(newTextData
== true) { textData
= TextData
.NewInstance(); }
269 //for diptera database
270 if (categoryFk
== 99 && notes
.contains("<OriginalName>")){
271 notes
= notes
.replaceAll("<OriginalName>", "");
272 notes
= notes
.replaceAll("</OriginalName>", "");
273 fact
= notes
+ ": " + fact
;
275 //TODO textData.putText(fact, bmiConfig.getFactLanguage()); //doesn't work because bmiConfig.getFactLanguage() is not not a persistent Language Object
276 //throws in thread "main" org.springframework.dao.InvalidDataAccessApiUsageException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language; nested exception is org.hibernate.TransientObjectException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language
278 textData
.addMedia(media
);
279 textData
.setType(Feature
.IMAGE());
281 textData
.putText(fact
, Language
.DEFAULT());
282 textData
.setType(feature
);
286 ReferenceBase citation
;
287 if (factRefFkObj
!= null){
288 citation
= referenceMap
.get(factRefFk
);
289 if (citation
== null){
290 citation
= nomRefMap
.get(factRefFk
);
292 if (citation
== null && (factRefFk
!= 0)){
293 logger
.warn("Citation not found in referenceMap: " + factRefFk
);
300 if (citation
!= null || CdmUtils
.isNotEmpty(details
)){
301 DescriptionElementSource originalSource
= DescriptionElementSource
.NewInstance();
302 originalSource
.setCitation(citation
);
303 originalSource
.setCitationMicroReference(details
);
304 textData
.addSource(originalSource
);
306 taxonDescription
.addElement(textData
);
309 textData
.addMarker(Marker
.NewInstance(MarkerType
.IS_DOUBTFUL(), true));
312 textData
.addMarker(Marker
.NewInstance(MarkerType
.PUBLISH(), publishFlag
));
314 Integer sequence
= rs
.getInt("Sequence");
315 if (sequence
!= null && sequence
!= 999){
316 String strSequence
= String
.valueOf(sequence
);
317 strSequence
= SEQUENCE_PREFIX
+ strSequence
;
318 //TODO make it an Extension when possible
319 //Extension datesExtension = Extension.NewInstance(textData, strSequence, ExtensionType.ORDER());
320 Annotation annotation
= Annotation
.NewInstance(strSequence
, Language
.DEFAULT());
321 textData
.addAnnotation(annotation
);
324 // if (categoryFkObj == FACT_DESCRIPTION){
326 // }else if (categoryFkObj == FACT_OBSERVATION){
328 // }else if (categoryFkObj == FACT_DISTRIBUTION_EM){
332 // //logger.warn("FactCategory " + categoryFk + " not yet implemented");
336 doCreatedUpdatedNotes(state
, textData
, rs
, "Fact");
339 //Designation References -> unclear how to map to CDM
340 //factId -> OriginalSource for descriptionElements not yet implemented
342 //sequence -> textData is not an identifiable entity therefore extensions are not possible
343 //fact category better
345 taxonStore
.add(taxon
);
347 } catch (Exception re
){
348 logger
.error("An exception occurred during the facts import");
353 logger
.info("Facts handled: " + (i
-1));
354 logger
.info("Taxa to save: " + taxonStore
.size());
355 getTaxonService().saveTaxonAll(taxonStore
);
357 logger
.info("end makeFacts ..." + getSuccessString(result
));
359 } catch (SQLException e
) {
360 logger
.error("SQLException:" + e
);
370 * @param descriptionSet
373 private TaxonDescription
makeImage(BerlinModelImportState state
, String fact
, Media media
, Set
<TaxonDescription
> descriptionSet
, Taxon taxon
) {
374 TaxonDescription taxonDescription
= null;
375 ReferenceBase sourceRef
= state
.getConfig().getSourceReference();
378 ImageMetaData imageMetaData
= new ImageMetaData();
381 url
= new URL(fact
.trim());
382 } catch (MalformedURLException e
) {
383 logger
.warn("Malformed URL. Image could not be imported: " + CdmUtils
.Nz(uri
));
386 imageMetaData
.readFrom(url
);
387 MediaRepresentation mediaRepresentation
= MediaRepresentation
.NewInstance(imageMetaData
.getMimeType(), null);
388 media
.addRepresentation(mediaRepresentation
);
389 ImageFile image
= ImageFile
.NewInstance(uri
, size
, imageMetaData
);
390 mediaRepresentation
.addRepresentationPart(image
);
392 taxonDescription
= taxon
.getOrCreateImageGallery(sourceRef
== null ?
null :sourceRef
.getTitleCache());
394 return taxonDescription
;
397 private TaxonBase
getTaxon(MapWrapper
<TaxonBase
> taxonMap
, Object taxonIdObj
, Integer taxonId
){
398 if (taxonIdObj
!= null){
399 return taxonMap
.get(taxonId
);
406 private Feature
getFeature(MapWrapper
<Feature
> featureMap
, Object categoryFkObj
, Integer categoryFk
){
407 if (categoryFkObj
!= null){
408 return featureMap
.get(categoryFk
);
415 private boolean checkDesignationRefsExist(BerlinModelImportConfigurator config
){
417 boolean result
= true;
418 Source source
= config
.getSource();
419 String strQueryArticlesWithoutJournal
= "SELECT Count(*) as n " +
421 " WHERE (NOT (PTDesignationRefFk IS NULL) ) OR " +
422 " (NOT (PTDesignationRefDetailFk IS NULL) )";
423 ResultSet rs
= source
.getResultSet(strQueryArticlesWithoutJournal
);
425 int count
= rs
.getInt("n");
427 System
.out
.println("========================================================");
428 logger
.warn("There are "+count
+" Facts with not empty designation references. Designation references are not imported.");
430 System
.out
.println("========================================================");
433 } catch (SQLException e
) {
441 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
443 protected boolean isIgnore(BerlinModelImportState state
){
444 return ! state
.getConfig().isDoFacts();