2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.berlinModel
.in
;
12 import java
.io
.IOException
;
14 import java
.net
.URISyntaxException
;
15 import java
.sql
.ResultSet
;
16 import java
.sql
.SQLException
;
17 import java
.util
.HashMap
;
18 import java
.util
.HashSet
;
19 import java
.util
.List
;
22 import java
.util
.UUID
;
24 import javax
.validation
.constraints
.NotNull
;
26 import org
.apache
.commons
.lang
.StringUtils
;
27 import org
.apache
.http
.HttpException
;
28 import org
.apache
.log4j
.Logger
;
29 import org
.springframework
.stereotype
.Component
;
31 import au
.com
.bytecode
.opencsv
.CSVReader
;
32 import eu
.etaxonomy
.cdm
.common
.CdmUtils
;
33 import eu
.etaxonomy
.cdm
.common
.UTF8
;
34 import eu
.etaxonomy
.cdm
.common
.media
.ImageInfo
;
35 import eu
.etaxonomy
.cdm
.database
.update
.DatabaseTypeNotSupportedException
;
36 import eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
;
37 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.validation
.BerlinModelFactsImportValidator
;
38 import eu
.etaxonomy
.cdm
.io
.common
.IOValidator
;
39 import eu
.etaxonomy
.cdm
.io
.common
.ResultSetPartitioner
;
40 import eu
.etaxonomy
.cdm
.io
.common
.Source
;
41 import eu
.etaxonomy
.cdm
.model
.agent
.AgentBase
;
42 import eu
.etaxonomy
.cdm
.model
.agent
.Person
;
43 import eu
.etaxonomy
.cdm
.model
.agent
.Team
;
44 import eu
.etaxonomy
.cdm
.model
.common
.Annotation
;
45 import eu
.etaxonomy
.cdm
.model
.common
.AnnotationType
;
46 import eu
.etaxonomy
.cdm
.model
.common
.CdmBase
;
47 import eu
.etaxonomy
.cdm
.model
.common
.IdentifiableSource
;
48 import eu
.etaxonomy
.cdm
.model
.common
.Language
;
49 import eu
.etaxonomy
.cdm
.model
.common
.LanguageString
;
50 import eu
.etaxonomy
.cdm
.model
.common
.Marker
;
51 import eu
.etaxonomy
.cdm
.model
.common
.MarkerType
;
52 import eu
.etaxonomy
.cdm
.model
.common
.OriginalSourceType
;
53 import eu
.etaxonomy
.cdm
.model
.common
.Representation
;
54 import eu
.etaxonomy
.cdm
.model
.common
.TermType
;
55 import eu
.etaxonomy
.cdm
.model
.common
.TermVocabulary
;
56 import eu
.etaxonomy
.cdm
.model
.description
.CommonTaxonName
;
57 import eu
.etaxonomy
.cdm
.model
.description
.DescriptionElementBase
;
58 import eu
.etaxonomy
.cdm
.model
.description
.DescriptionElementSource
;
59 import eu
.etaxonomy
.cdm
.model
.description
.Distribution
;
60 import eu
.etaxonomy
.cdm
.model
.description
.Feature
;
61 import eu
.etaxonomy
.cdm
.model
.description
.PresenceAbsenceTerm
;
62 import eu
.etaxonomy
.cdm
.model
.description
.TaxonDescription
;
63 import eu
.etaxonomy
.cdm
.model
.description
.TextData
;
64 import eu
.etaxonomy
.cdm
.model
.location
.Country
;
65 import eu
.etaxonomy
.cdm
.model
.location
.NamedArea
;
66 import eu
.etaxonomy
.cdm
.model
.media
.ImageFile
;
67 import eu
.etaxonomy
.cdm
.model
.media
.Media
;
68 import eu
.etaxonomy
.cdm
.model
.media
.MediaRepresentation
;
69 import eu
.etaxonomy
.cdm
.model
.media
.Rights
;
70 import eu
.etaxonomy
.cdm
.model
.media
.RightsType
;
71 import eu
.etaxonomy
.cdm
.model
.reference
.Reference
;
72 import eu
.etaxonomy
.cdm
.model
.reference
.ReferenceFactory
;
73 import eu
.etaxonomy
.cdm
.model
.taxon
.Taxon
;
74 import eu
.etaxonomy
.cdm
.model
.taxon
.TaxonBase
;
75 import eu
.etaxonomy
.cdm
.strategy
.exceptions
.UnknownCdmTypeException
;
76 import eu
.etaxonomy
.cdm
.strategy
.parser
.TimePeriodParser
;
83 public class BerlinModelFactsImport
extends BerlinModelImportBase
{
84 private static final long serialVersionUID
= 4095154818163504795L;
86 private static final Logger logger
= Logger
.getLogger(BerlinModelFactsImport
.class);
88 public static final String NAMESPACE
= "Fact";
90 public static final String SEQUENCE_PREFIX
= "ORDER: ";
92 private int modCount
= 10000;
93 private static final String pluralString
= "facts";
94 private static final String dbTableName
= "Fact";
96 //FIXME don't use as class variable
97 private Map
<Integer
, Feature
> featureMap
;
99 public BerlinModelFactsImport(){
100 super(dbTableName
, pluralString
);
104 private TermVocabulary
<Feature
> getFeatureVocabulary(){
105 TermVocabulary
<Feature
> newVoc
= TermVocabulary
.NewInstance(TermType
.Feature
, "User Defined Feature Vocabulary", "User Defined Feature Vocabulary", null, null);
106 getVocabularyService().save(newVoc
);
111 // //TODO work around until service method works
112 // TermVocabulary<Feature> featureVocabulary = BerlinModelTransformer.factCategory2Feature(1).getVocabulary();
113 // //TermVocabulary<Feature> vocabulary = getTermService().getVocabulary(vocabularyUuid);
114 // return featureVocabulary;
115 // } catch (UnknownCdmTypeException e) {
116 // logger.error("Feature vocabulary not available. New vocabulary created");
117 // return TermVocabulary.NewInstance(TermType.Feature, "User Defined Feature Vocabulary", "User Defined Feature Vocabulary", null, null);
121 private Map
<Integer
, Feature
> invokeFactCategories(BerlinModelImportState state
){
123 Map
<Integer
, Feature
> result
= state
.getConfig().getFeatureMap();
124 Source source
= state
.getConfig().getSource();
127 //get data from database
129 " SELECT FactCategory.* " +
130 " FROM FactCategory "+
132 if (state
.getConfig().isSalvador()){
133 strQuery
+= " AND " + state
.getConfig().getFactFilter().replace("factCategoryFk", "factCategoryId");
136 ResultSet rs
= source
.getResultSet(strQuery
) ;
138 TermVocabulary
<Feature
> featureVocabulary
= getFeatureVocabulary();
143 if ((i
++ % modCount
) == 0 && i
!= 1 ){ logger
.info("FactCategories handled: " + (i
-1));}
145 int factCategoryId
= rs
.getInt("factCategoryId");
146 String factCategory
= rs
.getString("factCategory");
150 feature
= BerlinModelTransformer
.factCategory2Feature(factCategoryId
);
151 } catch (UnknownCdmTypeException e
) {
152 UUID featureUuid
= null;
153 featureUuid
= BerlinModelTransformer
.getFeatureUuid(String
.valueOf(factCategoryId
+"-"+factCategory
));
154 if (featureUuid
== null){
155 logger
.warn("New Feature (FactCategoryId: " + factCategoryId
+ ")");
156 featureUuid
= UUID
.randomUUID();
158 feature
= getFeature(state
, featureUuid
, factCategory
, factCategory
, null, featureVocabulary
);
159 if (state
.getConfig().isSalvador()){
160 adaptNewSalvadorFeature(factCategoryId
, feature
);
163 doId(state
, feature
, factCategoryId
, "FactCategory");
166 // MaxFactNumber int Checked
167 // ExtensionTableName varchar(100) Checked
168 // Description nvarchar(1000) Checked
169 // locExtensionFormName nvarchar(80) Checked
170 // RankRestrictionFk int Checked
173 result
.put(factCategoryId
, feature
);
176 } catch (SQLException e
) {
177 logger
.error("SQLException:" + e
);
184 * @param factCategoryId
187 private void adaptNewSalvadorFeature(int factCategoryId
, Feature feature
) {
188 if (factCategoryId
== 306){
189 addSpanishFactCategoryName(feature
, "Nombre(s) común(es)");
190 } else if (factCategoryId
== 307){
191 addSpanishFactCategoryName(feature
, "Muestras de herbario");
192 } else if (factCategoryId
== 310){
193 addEnglishFactCategoryName(feature
, "Other references for taxon");
194 } else if (factCategoryId
== 309){
195 addEnglishFactCategoryName(feature
, "Report (reference) for El Salvador");
196 } else if (factCategoryId
== 311){
197 addEnglishFactCategoryName(feature
, "Taxon illustration references");
198 } else if (factCategoryId
== 312){
199 addSpanishFactCategoryName(feature
, "Imágen");
200 } else if (factCategoryId
== 350){
201 addSpanishFactCategoryName(feature
, "Descripción");
202 } else if (factCategoryId
== 303){
203 addEnglishFactCategoryName(feature
, "General distribution");
204 } else if (factCategoryId
== 2000){
205 addEnglishFactCategoryName(feature
, "Habitat in El Salvador");
206 } else if (factCategoryId
== 302){
207 addSpanishFactCategoryName(feature
, "Usos");
208 } else if (factCategoryId
== 1800){
209 addEnglishFactCategoryName(feature
, "Specimen notes");
210 } else if (factCategoryId
== 1900){
211 addEnglishFactCategoryName(feature
, "Editorial notes");
221 private void addSpanishFactCategoryName(Feature feature
, String label
) {
222 feature
.getRepresentations().add(Representation
.NewInstance(label
, label
, null, Language
.SPANISH_CASTILIAN()));
229 private void addEnglishFactCategoryName(Feature feature
, String label
) {
230 feature
.getRepresentations().iterator().next().setLanguage(Language
.SPANISH_CASTILIAN());
231 feature
.getRepresentations().add(Representation
.NewInstance(label
, label
, null, Language
.ENGLISH()));
236 protected void doInvoke(BerlinModelImportState state
) {
237 featureMap
= invokeFactCategories(state
);
238 super.doInvoke(state
);
244 protected String
getIdQuery(BerlinModelImportState state
) {
245 String result
= super.getIdQuery(state
);
246 if (StringUtils
.isNotBlank(state
.getConfig().getFactFilter())){
247 result
+= " WHERE " + state
.getConfig().getFactFilter();
249 result
= super.getIdQuery(state
);
251 result
+= getOrderBy(state
.getConfig());
256 protected String
getRecordQuery(BerlinModelImportConfigurator config
) {
258 " SELECT Fact.*, PTaxon.RIdentifier as taxonId, RefDetail.Details " +
260 " INNER JOIN PTaxon ON Fact.PTNameFk = PTaxon.PTNameFk AND Fact.PTRefFk = PTaxon.PTRefFk " +
261 " LEFT OUTER JOIN RefDetail ON Fact.FactRefDetailFk = RefDetail.RefDetailId AND Fact.FactRefFk = RefDetail.RefFk " +
262 " WHERE (FactId IN (" + ID_LIST_TOKEN
+ "))";
263 strQuery
+= getOrderBy(config
);
269 private String
getOrderBy(BerlinModelImportConfigurator config
) {
272 if (config
.getSource().checkColumnExists("Fact", "Sequence")){
273 result
= " ORDER By Fact.Sequence, Fact.FactId";
274 if (config
.isSalvador()){
275 result
= " ORDER By Fact.FactCategoryFk, Fact.Sequence, Fact.FactId";
278 result
= " ORDER By Fact.FactId";
280 } catch (DatabaseTypeNotSupportedException e
) {
281 logger
.info("checkColumnExists not supported");
282 result
= " ORDER By Fact.FactId";
288 public boolean doPartition(ResultSetPartitioner partitioner
, BerlinModelImportState state
) {
289 boolean success
= true ;
290 BerlinModelImportConfigurator config
= state
.getConfig();
291 Set
<TaxonBase
> taxaToSave
= new HashSet
<TaxonBase
>();
292 Map
<String
, TaxonBase
> taxonMap
= partitioner
.getObjectMap(BerlinModelTaxonImport
.NAMESPACE
);
293 Map
<String
, Reference
> refMap
= partitioner
.getObjectMap(BerlinModelReferenceImport
.REFERENCE_NAMESPACE
);
295 ResultSet rs
= partitioner
.getResultSet();
297 Reference sourceRef
= state
.getTransactionalSourceReference();
304 if ((i
++ % modCount
) == 0){ logger
.info("Facts handled: " + (i
-1));}
306 int factId
= rs
.getInt("factId");
307 Integer taxonId
= nullSafeInt(rs
, "taxonId");
308 Integer factRefFkInt
= nullSafeInt(rs
, "factRefFk");
309 Integer categoryFkInt
= nullSafeInt(rs
, "factCategoryFk");
310 String details
= rs
.getString("Details");
311 String fact
= CdmUtils
.Nz(rs
.getString("Fact"));
312 String notes
= CdmUtils
.Nz(rs
.getString("notes"));
313 Boolean doubtfulFlag
= rs
.getBoolean("DoubtfulFlag");
315 TaxonBase
<?
> taxonBase
= getTaxon(taxonMap
, taxonId
, taxonId
);
316 Feature feature
= getFeature(featureMap
, categoryFkInt
) ;
318 if (taxonBase
== null){
319 logger
.warn("Taxon for Fact " + factId
+ " does not exist in store");
322 TaxonDescription taxonDescription
;
323 if ( (taxonDescription
= getMyTaxonDescripion(taxonBase
, state
, categoryFkInt
, taxonId
, factId
, fact
, sourceRef
)) == null){
329 TextData textData
= null;
330 boolean newTextData
= true;
332 // For Cichorieae DB: If fact category is 31 (Systematics) and there is already a Systematics TextData
333 // description element append the fact text to the existing TextData
334 if(categoryFkInt
.equals(31)) {
335 Set
<DescriptionElementBase
> descriptionElements
= taxonDescription
.getElements();
336 for (DescriptionElementBase descriptionElement
: descriptionElements
) {
337 String featureString
= descriptionElement
.getFeature().getRepresentation(Language
.DEFAULT()).getLabel();
338 if (descriptionElement
instanceof TextData
&& featureString
.equals("Systematics")) { // TODO: test
339 textData
= (TextData
)descriptionElement
;
340 String factTextStr
= textData
.getText(Language
.DEFAULT());
341 // FIXME: Removing newlines doesn't work
342 if (factTextStr
.contains("\\r\\n")) {
343 factTextStr
= factTextStr
.replaceAll("\\r\\n","");
345 StringBuilder factText
= new StringBuilder(factTextStr
);
346 factText
.append(fact
);
347 fact
= factText
.toString();
354 if (taxonDescription
.isImageGallery()){
356 textData
= (TextData
)taxonDescription
.getElements().iterator().next();
358 if(newTextData
== true) {
359 textData
= TextData
.NewInstance();
363 //for diptera database
364 if (categoryFkInt
.equals(99) && notes
.contains("<OriginalName>")){
365 fact
= notes
+ ": " + fact
;
368 if (categoryFkInt
.equals(14) && state
.getConfig().isRemoveHttpMapsAnchor() && fact
.contains("<a href")){
369 //example <a href="http://euromed.luomus.fi/euromed_map.php?taxon=280629&size=medium">distribution</a>
370 fact
= fact
.replace("<a href=\"", "").replace("\">distribution</a>", "");
373 //TODO textData.putText(fact, bmiConfig.getFactLanguage()); //doesn't work because bmiConfig.getFactLanguage() is not not a persistent Language Object
374 //throws in thread "main" org.springframework.dao.InvalidDataAccessApiUsageException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language; nested exception is org.hibernate.TransientObjectException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language
376 Language lang
= Language
.DEFAULT();
377 if (state
.getConfig().isSalvador()){
378 lang
= getSalvadorFactLanguage(categoryFkInt
);
380 if (! taxonDescription
.isImageGallery()){
381 textData
.putText(lang
, fact
);
382 textData
.setFeature(feature
);
385 DescriptionElementBase deb
= textData
;
387 if (state
.getConfig().isSalvador()){
388 if (categoryFkInt
== 306){
389 deb
= CommonTaxonName
.NewInstance(fact
, Language
.SPANISH_CASTILIAN(), Country
.ELSALVADORREPUBLICOF());
390 }else if (categoryFkInt
== 307){
391 Distribution salvadorDistribution
= salvadorDistributionFromMuestrasDeHerbar((Taxon
)taxonBase
, fact
);
392 if (salvadorDistribution
!= null){
394 doId(state
, salvadorDistribution
, factId
, "Fact");
395 mergeSalvadorDistribution(taxonDescription
, salvadorDistribution
);
401 Reference citation
= null;
402 String factRefFk
= String
.valueOf(factRefFkInt
);
403 if (factRefFkInt
!= null){
404 citation
= refMap
.get(factRefFk
);
406 if (citation
== null && (factRefFkInt
!= null)){
407 logger
.warn("Citation not found in referenceMap: " + factRefFk
);
410 if (citation
!= null || StringUtils
.isNotBlank(details
)){
411 DescriptionElementSource originalSource
= DescriptionElementSource
.NewPrimarySourceInstance(citation
, details
);
412 deb
.addSource(originalSource
);
414 taxonDescription
.addElement(deb
);
417 deb
.addMarker(Marker
.NewInstance(MarkerType
.IS_DOUBTFUL(), true));
420 String strPublishFlag
= "publishFlag";
421 boolean publishFlagExists
= state
.getConfig().getSource().checkColumnExists(dbTableName
, strPublishFlag
);
422 if (publishFlagExists
){
423 Boolean publishFlag
= rs
.getBoolean(strPublishFlag
);
424 if (publishFlag
== false){
425 deb
.addMarker(Marker
.NewInstance(MarkerType
.PUBLISH(), publishFlag
));
430 Integer sequence
= rs
.getInt("Sequence");
431 if (sequence
!= 999){
432 String strSequence
= String
.valueOf(sequence
);
433 strSequence
= SEQUENCE_PREFIX
+ strSequence
;
434 //TODO make it an Extension when possible
435 //Extension datesExtension = Extension.NewInstance(textData, strSequence, ExtensionType.ORDER());
436 Annotation annotation
= Annotation
.NewInstance(strSequence
, AnnotationType
.TECHNICAL(), Language
.ENGLISH());
437 deb
.addAnnotation(annotation
);
440 // if (categoryFkObj == FACT_DESCRIPTION){
442 // }else if (categoryFkObj == FACT_OBSERVATION){
444 // }else if (categoryFkObj == FACT_DISTRIBUTION_EM){
448 // //logger.warn("FactCategory " + categoryFk + " not yet implemented");
452 doCreatedUpdatedNotes(state
, deb
, rs
);
453 doId(state
, deb
, factId
, "Fact");
456 //Designation References -> unclear how to map to CDM
459 //sequence -> textData is not an identifiable entity therefore extensions are not possible
460 //fact category better
462 taxaToSave
.add(taxonBase
);
464 } catch (Exception re
){
465 logger
.error("An exception occurred during the facts import");
466 re
.printStackTrace();
471 logger
.info("Facts handled: " + (i
-1));
472 logger
.info("Taxa to save: " + taxaToSave
.size());
473 getTaxonService().save(taxaToSave
);
474 }catch(SQLException e
){
475 throw new RuntimeException(e
);
482 * @param taxonDescription
483 * @param salvadorDistribution
485 private void mergeSalvadorDistribution(TaxonDescription taxonDescription
,
486 @NotNull Distribution newDistribution
) {
487 Distribution existingDistribution
= null;
488 for (DescriptionElementBase deb
: taxonDescription
.getElements()){
489 if (deb
.isInstanceOf(Distribution
.class)){
490 Distribution distribution
= CdmBase
.deproxy(deb
, Distribution
.class);
491 if (distribution
.getArea() != null && distribution
.getArea().equals(newDistribution
.getArea())){
492 existingDistribution
= distribution
;
494 }else if (distribution
.getArea() == null){
495 logger
.warn("Area for distribution is null: " + distribution
.getUuid());
499 if (existingDistribution
== null){
500 taxonDescription
.addElement(newDistribution
);
501 }else if(!existingDistribution
.getStatus().equals(newDistribution
.getStatus())){
503 logger
.warn("Taxon has areas with different distribution states: " + taxonDescription
.getTaxon().getTitleCache());
505 //do nothing, distribution already exists
510 private Map
<String
, NamedArea
> salvadorAreaMap
= null;
511 private Distribution
salvadorDistributionFromMuestrasDeHerbar(Taxon taxon
, String fact
) {
512 if (salvadorAreaMap
== null){
513 salvadorAreaMap
= new HashMap
<>();
514 TermVocabulary
<NamedArea
> salvadorAreas
= getVocabulary(TermType
.NamedArea
, BerlinModelTransformer
.uuidSalvadorAreas
,
515 "Salvador areas", "Salvador areas", null, null, true, NamedArea
.NewInstance());
516 getVocabularyService().save(salvadorAreas
);
518 Distribution result
= null;
519 String
[] areaStrings
= fact
.split(":");
520 if (areaStrings
.length
> 1){
521 String areaString
= areaStrings
[0];
522 NamedArea area
= salvadorAreaMap
.get(areaString
);
524 logger
.info("Added Salvador area: " + areaString
);
525 TermVocabulary
<NamedArea
> voc
= getVocabulary(TermType
.NamedArea
, BerlinModelTransformer
.uuidSalvadorAreas
,
526 "Salvador areas", "Salvador areas", null, null, true, NamedArea
.NewInstance());
527 NamedArea newArea
= NamedArea
.NewInstance(areaString
, areaString
, null);
528 newArea
.getRepresentations().iterator().next().setLanguage(Language
.SPANISH_CASTILIAN());
529 voc
.addTerm(newArea
);
530 getTermService().saveOrUpdate(newArea
);
531 salvadorAreaMap
.put(areaString
, newArea
);
533 PresenceAbsenceTerm state
= getSalvadorDistributionState(taxon
);
534 result
= Distribution
.NewInstance(area
, state
);
541 private PresenceAbsenceTerm
getSalvadorDistributionState(Taxon taxon
) {
542 boolean hasGlobalDist
= false;
543 for (TaxonDescription desc
: taxon
.getDescriptions()){
544 for (DescriptionElementBase deb
: desc
.getElements()){
545 if (deb
.getFeature().getUuid().equals(BerlinModelTransformer
.uuidFeatureDistributionGlobal
)){
546 hasGlobalDist
= true;
547 TextData textData
= CdmBase
.deproxy(deb
, TextData
.class);
548 for (LanguageString text
: textData
.getMultilanguageText().values()){
549 if (text
.getText().contains("El Salvador")){
550 return PresenceAbsenceTerm
.NATIVE();
557 logger
.warn("No global distribution found: " + taxon
.getTitleCache());
559 return hasGlobalDist ? PresenceAbsenceTerm
.CULTIVATED(): PresenceAbsenceTerm
.PRESENT();
567 private Language
getSalvadorFactLanguage(int categoryFkInt
) {
568 if (categoryFkInt
== 350){
569 return Language
.ENGLISH();
570 }else if (categoryFkInt
== 1800 || categoryFkInt
== 1900){
571 return Language
.UNDETERMINED();
573 return Language
.SPANISH_CASTILIAN();
577 private TaxonDescription
getMyTaxonDescripion(TaxonBase taxonBase
, BerlinModelImportState state
, Integer categoryFk
, Integer taxonId
, int factId
, String fact
, Reference sourceRef
) {
579 if ( taxonBase
instanceof Taxon
) {
580 taxon
= (Taxon
) taxonBase
;
582 logger
.warn("TaxonBase " + (taxonId
==null?
"(null)":taxonId
) + " for Fact " + factId
+ " was not of type Taxon but: " + taxonBase
.getClass().getSimpleName());
586 TaxonDescription taxonDescription
= null;
587 Set
<TaxonDescription
> descriptionSet
= taxon
.getDescriptions();
590 //for diptera / salvador images
591 if (categoryFk
== 51 || categoryFk
== 312){ //TODO check also FactCategory string
592 media
= Media
.NewInstance();
593 taxonDescription
= makeImage(state
, fact
, media
, descriptionSet
, taxon
);
595 if (taxonDescription
== null){
599 TextData textData
= null;
600 for (DescriptionElementBase el
: taxonDescription
.getElements()){
601 if (el
.isInstanceOf(TextData
.class)){
602 textData
= CdmBase
.deproxy(el
, TextData
.class);
605 if (textData
== null){
606 textData
= TextData
.NewInstance(Feature
.IMAGE());
607 taxonDescription
.addElement(textData
);
609 textData
.addMedia(media
);
611 //all others (no image) -> getDescription
613 boolean isPublic
= ! (categoryFk
== 1800 || categoryFk
== 1900 || categoryFk
== 2000);
614 for (TaxonDescription desc
: descriptionSet
){
616 if (! desc
.isImageGallery()){
617 if (state
.getConfig().isSalvador()){
618 if (desc
.isDefault() && isPublic
|| !desc
.isDefault() && !isPublic
){
619 taxonDescription
= desc
;
623 taxonDescription
= desc
;
628 if (taxonDescription
== null){
629 taxonDescription
= TaxonDescription
.NewInstance();
630 taxonDescription
.setTitleCache(sourceRef
== null ?
null : sourceRef
.getTitleCache(), true);
631 if (state
.getConfig().isSalvador()){
632 String title
= "Factual data for " + taxon
.getName().getTitleCache();
634 taxonDescription
.setDefault(isPublic
);
636 title
= "Non public f" + title
.substring(1);
638 taxonDescription
.setTitleCache(title
, true);
640 taxon
.addDescription(taxonDescription
);
643 return taxonDescription
;
648 public Map
<Object
, Map
<String
, ?
extends CdmBase
>> getRelatedObjectsForPartition(ResultSet rs
, BerlinModelImportState state
) {
652 Map
<Object
, Map
<String
, ?
extends CdmBase
>> result
= new HashMap
<Object
, Map
<String
, ?
extends CdmBase
>>();
655 Set
<String
> taxonIdSet
= new HashSet
<String
>();
656 Set
<String
> referenceIdSet
= new HashSet
<String
>();
657 Set
<String
> refDetailIdSet
= new HashSet
<String
>();
659 handleForeignKey(rs
, taxonIdSet
, "taxonId");
660 handleForeignKey(rs
, referenceIdSet
, "FactRefFk");
661 handleForeignKey(rs
, referenceIdSet
, "PTDesignationRefFk");
662 handleForeignKey(rs
, refDetailIdSet
, "FactRefDetailFk");
663 handleForeignKey(rs
, refDetailIdSet
, "PTDesignationRefDetailFk");
667 nameSpace
= BerlinModelTaxonImport
.NAMESPACE
;
668 cdmClass
= TaxonBase
.class;
670 Map
<String
, TaxonBase
> taxonMap
= (Map
<String
, TaxonBase
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
671 result
.put(nameSpace
, taxonMap
);
674 nameSpace
= BerlinModelReferenceImport
.REFERENCE_NAMESPACE
;
675 cdmClass
= Reference
.class;
676 idSet
= referenceIdSet
;
677 Map
<String
, Reference
> referenceMap
= (Map
<String
, Reference
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
678 result
.put(nameSpace
, referenceMap
);
681 nameSpace
= BerlinModelRefDetailImport
.REFDETAIL_NAMESPACE
;
682 cdmClass
= Reference
.class;
683 idSet
= refDetailIdSet
;
684 Map
<String
, Reference
> refDetailMap
= (Map
<String
, Reference
>)getCommonService().getSourcedObjectsByIdInSource(cdmClass
, idSet
, nameSpace
);
685 result
.put(nameSpace
, refDetailMap
);
687 } catch (SQLException e
) {
688 throw new RuntimeException(e
);
698 * @param descriptionSet
699 * @throws URISyntaxException
702 private TaxonDescription
makeImage(BerlinModelImportState state
, String fact
, Media media
, Set
<TaxonDescription
> descriptionSet
, Taxon taxon
) {
703 TaxonDescription taxonDescription
= null;
705 Reference sourceRef
= state
.getTransactionalSourceReference();
708 if (state
.getConfig().isSalvador()){
709 String thumbs
= "thumbs/";
710 String uriStrFormat
= "http://media.e-taxonomy.eu/salvador/berendsohn-et-al-%s/%s.jpg";
711 Integer intFact
= Integer
.valueOf(fact
);
713 int page
= intFact
+ 249;
716 page
= intFact
+ (intFact
< 403 ?
95 : 96);
717 }else if (intFact
>= 142){
719 page
= intFact
+ (intFact
< 255 ?
3 : 4);
721 String title
= getSalvadorImageTitle(intFact
);
722 media
.putTitle(Language
.SPANISH_CASTILIAN(), title
);
724 Reference ref
= getSalvadorReference(vol
);
725 String originalName
= getSalvadorImageNameInfo(intFact
);
726 IdentifiableSource source
= media
.addSource(OriginalSourceType
.PrimaryMediaSource
, fact
, "Fig.", ref
, String
.valueOf(page
));
727 source
.setOriginalNameString(originalName
);
728 media
.setArtist(getSalvadorArtist());
729 media
.addRights(getSalvadorCopyright(vol
));
730 String uriStr
= String
.format(uriStrFormat
, vol
, fact
);
731 String thumbUriStr
= String
.format(uriStrFormat
, vol
, thumbs
+ fact
);
732 uri
= new URI(uriStr
);
733 thumbUri
= new URI(thumbUriStr
);
735 uri
= new URI(fact
.trim());
739 makeMediaRepresentation(media
, uri
);
740 if (thumbUri
!= null){
741 makeMediaRepresentation(media
, thumbUri
);
744 taxonDescription
= taxon
.getOrCreateImageGallery(sourceRef
== null ?
null :sourceRef
.getTitleCache());
745 } catch (URISyntaxException e
) {
746 logger
.warn("URISyntaxException. Image could not be imported: " + fact
);
749 return taxonDescription
;
753 private Map
<Integer
, String
[]> salvadorImages
= null;
754 private String
getSalvadorImageTitle(Integer intFact
) {
755 initSalvadorImagesFile();
756 String
[] line
= salvadorImages
.get(intFact
);
758 logger
.warn("Could not find salvador image metadata for " + intFact
);
759 return String
.valueOf(intFact
);
762 String result
= CdmUtils
.concat(" " + UTF8
.EN_DASH
+ " ", line
[i
], line
[i
+ 1]);
767 private String
getSalvadorImageNameInfo(Integer intFact
) {
768 initSalvadorImagesFile();
769 String
[] line
= salvadorImages
.get(intFact
);
771 logger
.warn("Could not find salvador image metadata for " + intFact
);
772 return String
.valueOf(intFact
);
775 String result
= line
[i
].substring("Fig. ".length() + line
[0].length()).trim();
783 private void initSalvadorImagesFile() {
784 if (salvadorImages
== null){
785 salvadorImages
= new HashMap
<>();
787 CSVReader reader
= new CSVReader(CdmUtils
.getUtf8ResourceReader("salvador" + CdmUtils
.getFolderSeperator() + "SalvadorImages.csv"),';');
788 List
<String
[]> lines
= reader
.readAll();
789 for (String
[] line
: lines
){
790 if(! "ID".equals(line
[0])){
791 salvadorImages
.put(Integer
.valueOf(line
[0]), line
);
795 } catch (IOException e
) {
802 private Rights
getSalvadorCopyright(String vol
) {
804 if ("2009".equals(vol
)){
805 text
= "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2009.";
806 }else if ("2012".equals(vol
)){
807 text
= "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2012.";
808 }else if ("2016".equals(vol
)){
809 text
= "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2016.";
811 throw new RuntimeException("Volume not recognized: " + vol
);
813 Rights result
= Rights
.NewInstance(text
, Language
.SPANISH_CASTILIAN(), RightsType
.COPYRIGHT());
818 private Integer salvadorArtistId
;
819 private AgentBase
<?
> getSalvadorArtist() {
820 if (salvadorArtistId
== null){
821 Person person
= Person
.NewInstance();
822 person
.setFirstname("José Gerver");
823 person
.setLastname("Molina");
824 salvadorArtistId
= getAgentService().save(person
).getId();
827 return getAgentService().find(salvadorArtistId
);
831 private Integer salvadorRef1Id
;
832 private Integer salvadorRef2Id
;
833 private Integer salvadorRef3Id
;
835 private Reference
getSalvadorReference(String vol
){
836 if (salvadorRef1Id
== null){
837 makeSalvadorReferences();
839 if ("2009".equals(vol
)){
840 return getReferenceService().find(salvadorRef1Id
);
841 }else if ("2012".equals(vol
)){
842 return getReferenceService().find(salvadorRef2Id
);
843 }else if ("2016".equals(vol
)){
844 return getReferenceService().find(salvadorRef3Id
);
846 throw new RuntimeException("Volume not recognized: " + vol
);
851 private void makeSalvadorReferences() {
852 Person walter
= Person
.NewTitledInstance("Berendsohn, W. G.");
853 walter
.setFirstname("Walter G.");
854 walter
.setLastname("Berendsohn");
855 Person katja
= Person
.NewTitledInstance("Gruber, Anne Kathrina");
856 katja
.setFirstname("Anne Katharina");
857 katja
.setLastname("Gruber");
858 Person monte
= Person
.NewTitledInstance("Monterrosa Salomón, J.");
859 Person olmedo
= Person
.NewTitledInstance("Olmedo Galán, P.");
860 Person rodriguez
= Person
.NewTitledInstance("Rodríguez Delcid, D");
862 Team team1
= Team
.NewInstance();
863 team1
.addTeamMember(walter
);
864 team1
.addTeamMember(katja
);
865 team1
.addTeamMember(monte
);
867 Team team2
= Team
.NewInstance();
868 team2
.addTeamMember(walter
);
869 team2
.addTeamMember(katja
);
870 team2
.addTeamMember(rodriguez
);
871 team2
.addTeamMember(olmedo
);
873 Reference vol1
= ReferenceFactory
.newBook();
874 Reference vol2
= ReferenceFactory
.newBook();
875 Reference vol3
= ReferenceFactory
.newBook();
877 vol1
.setAuthorship(team1
);
878 vol2
.setAuthorship(team1
);
879 vol3
.setAuthorship(team2
);
881 vol1
.setDatePublished(TimePeriodParser
.parseString("2009"));
882 vol2
.setDatePublished(TimePeriodParser
.parseString("2012"));
883 vol3
.setDatePublished(TimePeriodParser
.parseString("2016"));
885 Reference englera
= ReferenceFactory
.newPrintSeries("Englera");
886 vol1
.setInSeries(englera
);
887 vol2
.setInSeries(englera
);
888 vol3
.setInSeries(englera
);
890 vol1
.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 1: Angiospermae - Familias A-L");
891 vol2
.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 2: Angiospermae - Familias M-P y Pteridophyta");
892 vol3
.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 3: Angiospermae - Familias R-Z y Gymnospermae");
894 vol1
.setVolume("29(1)");
895 vol2
.setVolume("29(2)");
896 vol3
.setVolume("29(3)");
898 vol1
.setPages("1-438");
899 vol2
.setVolume("1-300");
900 vol3
.setVolume("1-356");
902 salvadorRef1Id
= getReferenceService().save(vol1
).getId();
903 salvadorRef2Id
= getReferenceService().find(getReferenceService().saveOrUpdate(vol2
)).getId();
904 salvadorRef3Id
= getReferenceService().find(getReferenceService().saveOrUpdate(vol3
)).getId();
915 private void makeMediaRepresentation(Media media
, URI uri
) {
916 ImageInfo imageInfo
= null;
919 imageInfo
= ImageInfo
.NewInstance(uri
, 0);
920 } catch (IOException
| HttpException e
) {
921 logger
.error("Error when reading image meta: " + e
+ ", "+ uri
.toString());
923 String mimeType
= imageInfo
== null ?
null : imageInfo
.getMimeType();
924 String suffix
= imageInfo
== null ?
null : imageInfo
.getSuffix();
925 MediaRepresentation mediaRepresentation
= MediaRepresentation
.NewInstance(mimeType
, suffix
);
926 media
.addRepresentation(mediaRepresentation
);
927 ImageFile image
= ImageFile
.NewInstance(uri
, size
, imageInfo
);
928 mediaRepresentation
.addRepresentationPart(image
);
931 private TaxonBase
<?
> getTaxon(Map
<String
, TaxonBase
> taxonMap
, Integer taxonIdObj
, Number taxonId
){
932 if (taxonIdObj
!= null){
933 return taxonMap
.get(String
.valueOf(taxonId
));
940 private Feature
getFeature(Map
<Integer
, Feature
> featureMap
, Integer categoryFkInt
){
941 if (categoryFkInt
!= null){
942 return featureMap
.get(categoryFkInt
);
950 protected boolean doCheck(BerlinModelImportState state
){
951 IOValidator
<BerlinModelImportState
> validator
= new BerlinModelFactsImportValidator();
952 return validator
.validate(state
);
956 protected boolean isIgnore(BerlinModelImportState state
){
957 return ! state
.getConfig().isDoFacts();