BGBM BDI Projects / cdmlib-apps.git / blob
? search:


2bd4b8cc96d1e1bb4be17745e68766ac74757415
[cdmlib-apps.git] / app-import / src / main / java / eu / etaxonomy / cdm / io / berlinModel / in / BerlinModelFactsImport.java
1 /**
2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
5 *
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
8 */
9
10 package eu.etaxonomy.cdm.io.berlinModel.in;
11
12 import java.io.IOException;
13 import java.net.URI;
14 import java.net.URISyntaxException;
15 import java.sql.ResultSet;
16 import java.sql.SQLException;
17 import java.util.HashMap;
18 import java.util.HashSet;
19 import java.util.List;
20 import java.util.Map;
21 import java.util.Set;
22 import java.util.UUID;
23
24 import javax.validation.constraints.NotNull;
25
26 import org.apache.commons.lang.StringUtils;
27 import org.apache.http.HttpException;
28 import org.apache.log4j.Logger;
29 import org.springframework.stereotype.Component;
30
31 import au.com.bytecode.opencsv.CSVReader;
32 import eu.etaxonomy.cdm.common.CdmUtils;
33 import eu.etaxonomy.cdm.common.UTF8;
34 import eu.etaxonomy.cdm.common.media.ImageInfo;
35 import eu.etaxonomy.cdm.database.update.DatabaseTypeNotSupportedException;
36 import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
37 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelFactsImportValidator;
38 import eu.etaxonomy.cdm.io.common.IOValidator;
39 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
40 import eu.etaxonomy.cdm.io.common.Source;
41 import eu.etaxonomy.cdm.model.agent.AgentBase;
42 import eu.etaxonomy.cdm.model.agent.Person;
43 import eu.etaxonomy.cdm.model.agent.Team;
44 import eu.etaxonomy.cdm.model.common.Annotation;
45 import eu.etaxonomy.cdm.model.common.AnnotationType;
46 import eu.etaxonomy.cdm.model.common.CdmBase;
47 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
48 import eu.etaxonomy.cdm.model.common.Language;
49 import eu.etaxonomy.cdm.model.common.LanguageString;
50 import eu.etaxonomy.cdm.model.common.Marker;
51 import eu.etaxonomy.cdm.model.common.MarkerType;
52 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
53 import eu.etaxonomy.cdm.model.common.Representation;
54 import eu.etaxonomy.cdm.model.common.TermType;
55 import eu.etaxonomy.cdm.model.common.TermVocabulary;
56 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
57 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
58 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
59 import eu.etaxonomy.cdm.model.description.Distribution;
60 import eu.etaxonomy.cdm.model.description.Feature;
61 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
62 import eu.etaxonomy.cdm.model.description.TaxonDescription;
63 import eu.etaxonomy.cdm.model.description.TextData;
64 import eu.etaxonomy.cdm.model.location.Country;
65 import eu.etaxonomy.cdm.model.location.NamedArea;
66 import eu.etaxonomy.cdm.model.media.ImageFile;
67 import eu.etaxonomy.cdm.model.media.Media;
68 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
69 import eu.etaxonomy.cdm.model.media.Rights;
70 import eu.etaxonomy.cdm.model.media.RightsType;
71 import eu.etaxonomy.cdm.model.reference.Reference;
72 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
73 import eu.etaxonomy.cdm.model.taxon.Taxon;
74 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
75 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
76 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
77
78 /**
79 * @author a.mueller
80 * @created 20.03.2008
81 */
82 @Component
83 public class BerlinModelFactsImport extends BerlinModelImportBase {
84 private static final long serialVersionUID = 4095154818163504795L;
85
86 private static final Logger logger = Logger.getLogger(BerlinModelFactsImport.class);
87
88 public static final String NAMESPACE = "Fact";
89
90 public static final String SEQUENCE_PREFIX = "ORDER: ";
91
92 private int modCount = 10000;
93 private static final String pluralString = "facts";
94 private static final String dbTableName = "Fact";
95
96 //FIXME don't use as class variable
97 private Map<Integer, Feature> featureMap;
98
99 public BerlinModelFactsImport(){
100 super(dbTableName, pluralString);
101 }
102
103
104 private TermVocabulary<Feature> getFeatureVocabulary(){
105 TermVocabulary<Feature> newVoc = TermVocabulary.NewInstance(TermType.Feature, "User Defined Feature Vocabulary", "User Defined Feature Vocabulary", null, null);
106 getVocabularyService().save(newVoc);
107
108 return newVoc;
109
110 // try {
111 // //TODO work around until service method works
112 // TermVocabulary<Feature> featureVocabulary = BerlinModelTransformer.factCategory2Feature(1).getVocabulary();
113 // //TermVocabulary<Feature> vocabulary = getTermService().getVocabulary(vocabularyUuid);
114 // return featureVocabulary;
115 // } catch (UnknownCdmTypeException e) {
116 // logger.error("Feature vocabulary not available. New vocabulary created");
117 // return TermVocabulary.NewInstance(TermType.Feature, "User Defined Feature Vocabulary", "User Defined Feature Vocabulary", null, null);
118 // }
119 }
120
121 private Map<Integer, Feature> invokeFactCategories(BerlinModelImportState state){
122
123 Map<Integer, Feature> result = state.getConfig().getFeatureMap();
124 Source source = state.getConfig().getSource();
125
126 try {
127 //get data from database
128 String strQuery =
129 " SELECT FactCategory.* " +
130 " FROM FactCategory "+
131 " WHERE (1=1)";
132 if (state.getConfig().isSalvador()){
133 strQuery += " AND " + state.getConfig().getFactFilter().replace("factCategoryFk", "factCategoryId");
134 }
135
136 ResultSet rs = source.getResultSet(strQuery) ;
137
138 TermVocabulary<Feature> featureVocabulary = getFeatureVocabulary();
139 int i = 0;
140 //for each reference
141 while (rs.next()){
142
143 if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("FactCategories handled: " + (i-1));}
144
145 int factCategoryId = rs.getInt("factCategoryId");
146 String factCategory = rs.getString("factCategory");
147
148 Feature feature;
149 try {
150 feature = BerlinModelTransformer.factCategory2Feature(factCategoryId);
151 } catch (UnknownCdmTypeException e) {
152 UUID featureUuid = null;
153 featureUuid = BerlinModelTransformer.getFeatureUuid(String.valueOf(factCategoryId+"-"+factCategory));
154 if (featureUuid == null){
155 logger.warn("New Feature (FactCategoryId: " + factCategoryId + ")");
156 featureUuid = UUID.randomUUID();
157 }
158 feature = getFeature(state, featureUuid, factCategory, factCategory, null, featureVocabulary);
159 if (state.getConfig().isSalvador()){
160 adaptNewSalvadorFeature(factCategoryId, feature);
161 }
162 //id
163 doId(state, feature, factCategoryId, "FactCategory");
164
165 //TODO
166 // MaxFactNumber int Checked
167 // ExtensionTableName varchar(100) Checked
168 // Description nvarchar(1000) Checked
169 // locExtensionFormName nvarchar(80) Checked
170 // RankRestrictionFk int Checked
171 }
172
173 result.put(factCategoryId, feature);
174 }
175 return result;
176 } catch (SQLException e) {
177 logger.error("SQLException:" + e);
178 return null;
179 }
180 }
181
182
183 /**
184 * @param factCategoryId
185 * @param feature
186 */
187 private void adaptNewSalvadorFeature(int factCategoryId, Feature feature) {
188 if (factCategoryId == 306){
189 addSpanishFactCategoryName(feature, "Nombre(s) común(es)");
190 } else if (factCategoryId == 307){
191 addSpanishFactCategoryName(feature, "Muestras de herbario");
192 } else if (factCategoryId == 310){
193 addEnglishFactCategoryName(feature, "Other references for taxon");
194 } else if (factCategoryId == 309){
195 addEnglishFactCategoryName(feature, "Report (reference) for El Salvador");
196 } else if (factCategoryId == 311){
197 addEnglishFactCategoryName(feature, "Taxon illustration references");
198 } else if (factCategoryId == 312){
199 addSpanishFactCategoryName(feature, "Imágen");
200 } else if (factCategoryId == 350){
201 addSpanishFactCategoryName(feature, "Descripción");
202 } else if (factCategoryId == 303){
203 addEnglishFactCategoryName(feature, "General distribution");
204 } else if (factCategoryId == 2000){
205 addEnglishFactCategoryName(feature, "Habitat in El Salvador");
206 } else if (factCategoryId == 302){
207 addSpanishFactCategoryName(feature, "Usos");
208 } else if (factCategoryId == 1800){
209 addEnglishFactCategoryName(feature, "Specimen notes");
210 } else if (factCategoryId == 1900){
211 addEnglishFactCategoryName(feature, "Editorial notes");
212 }
213
214 }
215
216
217 /**
218 * @param feature
219 * @param string
220 */
221 private void addSpanishFactCategoryName(Feature feature, String label) {
222 feature.getRepresentations().add(Representation.NewInstance(label, label, null, Language.SPANISH_CASTILIAN()));
223 }
224
225 /**
226 * @param feature
227 * @param string
228 */
229 private void addEnglishFactCategoryName(Feature feature, String label) {
230 feature.getRepresentations().iterator().next().setLanguage(Language.SPANISH_CASTILIAN());
231 feature.getRepresentations().add(Representation.NewInstance(label, label, null, Language.ENGLISH()));
232 }
233
234
235 @Override
236 protected void doInvoke(BerlinModelImportState state) {
237 featureMap = invokeFactCategories(state);
238 super.doInvoke(state);
239 return;
240 }
241
242
243 @Override
244 protected String getIdQuery(BerlinModelImportState state) {
245 String result = super.getIdQuery(state);
246 if (StringUtils.isNotBlank(state.getConfig().getFactFilter())){
247 result += " WHERE " + state.getConfig().getFactFilter();
248 }else{
249 result = super.getIdQuery(state);
250 }
251 result += getOrderBy(state.getConfig());
252 return result;
253 }
254
255 @Override
256 protected String getRecordQuery(BerlinModelImportConfigurator config) {
257 String strQuery =
258 " SELECT Fact.*, PTaxon.RIdentifier as taxonId, RefDetail.Details " +
259 " FROM Fact " +
260 " INNER JOIN PTaxon ON Fact.PTNameFk = PTaxon.PTNameFk AND Fact.PTRefFk = PTaxon.PTRefFk " +
261 " LEFT OUTER JOIN RefDetail ON Fact.FactRefDetailFk = RefDetail.RefDetailId AND Fact.FactRefFk = RefDetail.RefFk " +
262 " WHERE (FactId IN (" + ID_LIST_TOKEN + "))";
263 strQuery += getOrderBy(config);
264
265 return strQuery;
266 }
267
268
269 private String getOrderBy(BerlinModelImportConfigurator config) {
270 String result;
271 try{
272 if (config.getSource().checkColumnExists("Fact", "Sequence")){
273 result = " ORDER By Fact.Sequence, Fact.FactId";
274 if (config.isSalvador()){
275 result = " ORDER By Fact.FactCategoryFk, Fact.Sequence, Fact.FactId";
276 }
277 }else{
278 result = " ORDER By Fact.FactId";
279 }
280 } catch (DatabaseTypeNotSupportedException e) {
281 logger.info("checkColumnExists not supported");
282 result = " ORDER By Fact.FactId";
283 }
284 return result;
285 }
286
287 @Override
288 public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
289 boolean success = true ;
290 BerlinModelImportConfigurator config = state.getConfig();
291 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
292 Map<String, TaxonBase> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
293 Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
294
295 ResultSet rs = partitioner.getResultSet();
296
297 Reference sourceRef = state.getTransactionalSourceReference();
298
299 try{
300 int i = 0;
301 //for each fact
302 while (rs.next()){
303 try{
304 if ((i++ % modCount) == 0){ logger.info("Facts handled: " + (i-1));}
305
306 int factId = rs.getInt("factId");
307 Integer taxonId = nullSafeInt(rs, "taxonId");
308 Integer factRefFkInt = nullSafeInt(rs, "factRefFk");
309 Integer categoryFkInt = nullSafeInt(rs, "factCategoryFk");
310 String details = rs.getString("Details");
311 String fact = CdmUtils.Nz(rs.getString("Fact"));
312 String notes = CdmUtils.Nz(rs.getString("notes"));
313 Boolean doubtfulFlag = rs.getBoolean("DoubtfulFlag");
314
315 TaxonBase<?> taxonBase = getTaxon(taxonMap, taxonId, taxonId);
316 Feature feature = getFeature(featureMap, categoryFkInt) ;
317
318 if (taxonBase == null){
319 logger.warn("Taxon for Fact " + factId + " does not exist in store");
320 success = false;
321 }else{
322 TaxonDescription taxonDescription;
323 if ( (taxonDescription = getMyTaxonDescripion(taxonBase, state, categoryFkInt, taxonId, factId, fact, sourceRef)) == null){
324 success = false;
325 continue;
326 }
327
328 //textData
329 TextData textData = null;
330 boolean newTextData = true;
331
332 // For Cichorieae DB: If fact category is 31 (Systematics) and there is already a Systematics TextData
333 // description element append the fact text to the existing TextData
334 if(categoryFkInt.equals(31)) {
335 Set<DescriptionElementBase> descriptionElements = taxonDescription.getElements();
336 for (DescriptionElementBase descriptionElement : descriptionElements) {
337 String featureString = descriptionElement.getFeature().getRepresentation(Language.DEFAULT()).getLabel();
338 if (descriptionElement instanceof TextData && featureString.equals("Systematics")) { // TODO: test
339 textData = (TextData)descriptionElement;
340 String factTextStr = textData.getText(Language.DEFAULT());
341 // FIXME: Removing newlines doesn't work
342 if (factTextStr.contains("\\r\\n")) {
343 factTextStr = factTextStr.replaceAll("\\r\\n","");
344 }
345 StringBuilder factText = new StringBuilder(factTextStr);
346 factText.append(fact);
347 fact = factText.toString();
348 newTextData = false;
349 break;
350 }
351 }
352 }
353
354 if (taxonDescription.isImageGallery()){
355 newTextData = false;
356 textData = (TextData)taxonDescription.getElements().iterator().next();
357 }
358 if(newTextData == true) {
359 textData = TextData.NewInstance();
360 }
361
362
363 //for diptera database
364 if (categoryFkInt.equals(99) && notes.contains("<OriginalName>")){
365 fact = notes + ": " + fact ;
366 }
367 //for E+M maps
368 if (categoryFkInt.equals(14) && state.getConfig().isRemoveHttpMapsAnchor() && fact.contains("<a href")){
369 //example <a href="http://euromed.luomus.fi/euromed_map.php?taxon=280629&size=medium">distribution</a>
370 fact = fact.replace("<a href=\"", "").replace("\">distribution</a>", "");
371 }
372
373 //TODO textData.putText(fact, bmiConfig.getFactLanguage()); //doesn't work because bmiConfig.getFactLanguage() is not not a persistent Language Object
374 //throws in thread "main" org.springframework.dao.InvalidDataAccessApiUsageException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language; nested exception is org.hibernate.TransientObjectException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language
375
376 Language lang = Language.DEFAULT();
377 if (state.getConfig().isSalvador()){
378 lang = getSalvadorFactLanguage(categoryFkInt);
379 }
380 if (! taxonDescription.isImageGallery()){
381 textData.putText(lang, fact);
382 textData.setFeature(feature);
383 }
384
385 DescriptionElementBase deb = textData;
386
387 if (state.getConfig().isSalvador()){
388 if (categoryFkInt == 306){
389 deb = CommonTaxonName.NewInstance(fact, Language.SPANISH_CASTILIAN(), Country.ELSALVADORREPUBLICOF());
390 }else if (categoryFkInt == 307){
391 Distribution salvadorDistribution = salvadorDistributionFromMuestrasDeHerbar((Taxon)taxonBase, fact);
392 if (salvadorDistribution != null){
393 //id
394 doId(state, salvadorDistribution, factId, "Fact");
395 mergeSalvadorDistribution(taxonDescription, salvadorDistribution);
396 }
397 }
398 }
399
400 //reference
401 Reference citation = null;
402 String factRefFk = String.valueOf(factRefFkInt);
403 if (factRefFkInt != null){
404 citation = refMap.get(factRefFk);
405 }
406 if (citation == null && (factRefFkInt != null)){
407 logger.warn("Citation not found in referenceMap: " + factRefFk);
408 success = false;
409 }
410 if (citation != null || StringUtils.isNotBlank(details)){
411 DescriptionElementSource originalSource = DescriptionElementSource.NewPrimarySourceInstance(citation, details);
412 deb.addSource(originalSource);
413 }
414 taxonDescription.addElement(deb);
415 //doubtfulFlag
416 if (doubtfulFlag){
417 deb.addMarker(Marker.NewInstance(MarkerType.IS_DOUBTFUL(), true));
418 }
419 //publisheFlag
420 String strPublishFlag = "publishFlag";
421 boolean publishFlagExists = state.getConfig().getSource().checkColumnExists(dbTableName, strPublishFlag);
422 if (publishFlagExists){
423 Boolean publishFlag = rs.getBoolean(strPublishFlag);
424 if (publishFlag == false){
425 deb.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), publishFlag));
426 }
427 }
428
429 //Sequence
430 Integer sequence = rs.getInt("Sequence");
431 if (sequence != 999){
432 String strSequence = String.valueOf(sequence);
433 strSequence = SEQUENCE_PREFIX + strSequence;
434 //TODO make it an Extension when possible
435 //Extension datesExtension = Extension.NewInstance(textData, strSequence, ExtensionType.ORDER());
436 Annotation annotation = Annotation.NewInstance(strSequence, AnnotationType.TECHNICAL(), Language.ENGLISH());
437 deb.addAnnotation(annotation);
438 }
439
440 // if (categoryFkObj == FACT_DESCRIPTION){
441 // //;
442 // }else if (categoryFkObj == FACT_OBSERVATION){
443 // //;
444 // }else if (categoryFkObj == FACT_DISTRIBUTION_EM){
445 // //
446 // }else {
447 // //TODO
448 // //logger.warn("FactCategory " + categoryFk + " not yet implemented");
449 // }
450
451 //notes
452 doCreatedUpdatedNotes(state, deb, rs);
453 doId(state, deb, factId, "Fact");
454
455 //TODO
456 //Designation References -> unclear how to map to CDM
457
458
459 //sequence -> textData is not an identifiable entity therefore extensions are not possible
460 //fact category better
461
462 taxaToSave.add(taxonBase);
463 }
464 } catch (Exception re){
465 logger.error("An exception occurred during the facts import");
466 re.printStackTrace();
467 success = false;
468 }
469 //put
470 }
471 logger.info("Facts handled: " + (i-1));
472 logger.info("Taxa to save: " + taxaToSave.size());
473 getTaxonService().save(taxaToSave);
474 }catch(SQLException e){
475 throw new RuntimeException(e);
476 }
477 return success;
478 }
479
480
481 /**
482 * @param taxonDescription
483 * @param salvadorDistribution
484 */
485 private void mergeSalvadorDistribution(TaxonDescription taxonDescription,
486 @NotNull Distribution newDistribution) {
487 Distribution existingDistribution = null;
488 for (DescriptionElementBase deb : taxonDescription.getElements()){
489 if (deb.isInstanceOf(Distribution.class)){
490 Distribution distribution = CdmBase.deproxy(deb, Distribution.class);
491 if (distribution.getArea() != null && distribution.getArea().equals(newDistribution.getArea())){
492 existingDistribution = distribution;
493 break;
494 }else if (distribution.getArea() == null){
495 logger.warn("Area for distribution is null: " + distribution.getUuid());
496 }
497 }
498 }
499 if (existingDistribution == null){
500 taxonDescription.addElement(newDistribution);
501 }else if(!existingDistribution.getStatus().equals(newDistribution.getStatus())){
502 //should not happen
503 logger.warn("Taxon has areas with different distribution states: " + taxonDescription.getTaxon().getTitleCache());
504 }else{
505 //do nothing, distribution already exists
506 }
507 }
508
509
510 private Map<String, NamedArea> salvadorAreaMap = null;
511 private Distribution salvadorDistributionFromMuestrasDeHerbar(Taxon taxon, String fact) {
512 if (salvadorAreaMap == null){
513 salvadorAreaMap = new HashMap<>();
514 TermVocabulary<NamedArea> salvadorAreas = getVocabulary(TermType.NamedArea, BerlinModelTransformer.uuidSalvadorAreas,
515 "Salvador areas", "Salvador areas", null, null, true, NamedArea.NewInstance());
516 getVocabularyService().save(salvadorAreas);
517 }
518 Distribution result = null;
519 String[] areaStrings = fact.split(":");
520 if (areaStrings.length > 1){
521 String areaString = areaStrings[0];
522 NamedArea area = salvadorAreaMap.get(areaString);
523 if (area == null){
524 logger.info("Added Salvador area: " + areaString);
525 TermVocabulary<NamedArea> voc = getVocabulary(TermType.NamedArea, BerlinModelTransformer.uuidSalvadorAreas,
526 "Salvador areas", "Salvador areas", null, null, true, NamedArea.NewInstance());
527 NamedArea newArea = NamedArea.NewInstance(areaString, areaString, null);
528 newArea.getRepresentations().iterator().next().setLanguage(Language.SPANISH_CASTILIAN());
529 voc.addTerm(newArea);
530 getTermService().saveOrUpdate(newArea);
531 salvadorAreaMap.put(areaString, newArea);
532 }
533 PresenceAbsenceTerm state = getSalvadorDistributionState(taxon);
534 result = Distribution.NewInstance(area, state);
535 return result;
536 }else{
537 return null;
538 }
539 }
540
541 private PresenceAbsenceTerm getSalvadorDistributionState(Taxon taxon) {
542 boolean hasGlobalDist = false;
543 for (TaxonDescription desc : taxon.getDescriptions()){
544 for (DescriptionElementBase deb : desc.getElements()){
545 if (deb.getFeature().getUuid().equals(BerlinModelTransformer.uuidFeatureDistributionGlobal)){
546 hasGlobalDist = true;
547 TextData textData = CdmBase.deproxy(deb, TextData.class);
548 for (LanguageString text : textData.getMultilanguageText().values()){
549 if (text.getText().contains("El Salvador")){
550 return PresenceAbsenceTerm.NATIVE();
551 }
552 }
553 }
554 }
555 }
556 if (!hasGlobalDist){
557 logger.warn("No global distribution found: " + taxon.getTitleCache());
558 }
559 return hasGlobalDist ? PresenceAbsenceTerm.CULTIVATED(): PresenceAbsenceTerm.PRESENT();
560 }
561
562
563 /**
564 * @param factId
565 * @return
566 */
567 private Language getSalvadorFactLanguage(int categoryFkInt) {
568 if (categoryFkInt == 350){
569 return Language.ENGLISH();
570 }else if (categoryFkInt == 1800 || categoryFkInt == 1900){
571 return Language.UNDETERMINED();
572 }
573 return Language.SPANISH_CASTILIAN();
574 }
575
576
577 private TaxonDescription getMyTaxonDescripion(TaxonBase taxonBase, BerlinModelImportState state, Integer categoryFk, Integer taxonId, int factId, String fact, Reference sourceRef) {
578 Taxon taxon = null;
579 if ( taxonBase instanceof Taxon ) {
580 taxon = (Taxon) taxonBase;
581 }else{
582 logger.warn("TaxonBase " + (taxonId==null?"(null)":taxonId) + " for Fact " + factId + " was not of type Taxon but: " + taxonBase.getClass().getSimpleName());
583 return null;
584 }
585
586 TaxonDescription taxonDescription = null;
587 Set<TaxonDescription> descriptionSet= taxon.getDescriptions();
588
589 Media media = null;
590 //for diptera / salvador images
591 if (categoryFk == 51 || categoryFk == 312){ //TODO check also FactCategory string
592 media = Media.NewInstance();
593 taxonDescription = makeImage(state, fact, media, descriptionSet, taxon);
594
595 if (taxonDescription == null){
596 return null;
597 }
598
599 TextData textData = null;
600 for (DescriptionElementBase el: taxonDescription.getElements()){
601 if (el.isInstanceOf(TextData.class)){
602 textData = CdmBase.deproxy(el, TextData.class);
603 }
604 }
605 if (textData == null){
606 textData = TextData.NewInstance(Feature.IMAGE());
607 taxonDescription.addElement(textData);
608 }
609 textData.addMedia(media);
610 }
611 //all others (no image) -> getDescription
612 else{
613 boolean isPublic = ! (categoryFk == 1800 || categoryFk == 1900 || categoryFk == 2000);
614 for (TaxonDescription desc: descriptionSet){
615
616 if (! desc.isImageGallery()){
617 if (state.getConfig().isSalvador()){
618 if (desc.isDefault() && isPublic || !desc.isDefault() && !isPublic){
619 taxonDescription = desc;
620 break;
621 }
622 }else{
623 taxonDescription = desc;
624 break;
625 }
626 }
627 }
628 if (taxonDescription == null){
629 taxonDescription = TaxonDescription.NewInstance();
630 taxonDescription.setTitleCache(sourceRef == null ? null : sourceRef.getTitleCache(), true);
631 if (state.getConfig().isSalvador()){
632 String title = "Factual data for " + taxon.getName().getTitleCache();
633 if (isPublic){
634 taxonDescription.setDefault(isPublic);
635 }else{
636 title = "Non public f" + title.substring(1);
637 }
638 taxonDescription.setTitleCache(title, true);
639 }
640 taxon.addDescription(taxonDescription);
641 }
642 }
643 return taxonDescription;
644 }
645
646
647 @Override
648 public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
649 String nameSpace;
650 Class<?> cdmClass;
651 Set<String> idSet;
652 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
653
654 try{
655 Set<String> taxonIdSet = new HashSet<String>();
656 Set<String> referenceIdSet = new HashSet<String>();
657 Set<String> refDetailIdSet = new HashSet<String>();
658 while (rs.next()){
659 handleForeignKey(rs, taxonIdSet, "taxonId");
660 handleForeignKey(rs, referenceIdSet, "FactRefFk");
661 handleForeignKey(rs, referenceIdSet, "PTDesignationRefFk");
662 handleForeignKey(rs, refDetailIdSet, "FactRefDetailFk");
663 handleForeignKey(rs, refDetailIdSet, "PTDesignationRefDetailFk");
664 }
665
666 //taxon map
667 nameSpace = BerlinModelTaxonImport.NAMESPACE;
668 cdmClass = TaxonBase.class;
669 idSet = taxonIdSet;
670 Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
671 result.put(nameSpace, taxonMap);
672
673 //reference map
674 nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
675 cdmClass = Reference.class;
676 idSet = referenceIdSet;
677 Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
678 result.put(nameSpace, referenceMap);
679
680 //refDetail map
681 nameSpace = BerlinModelRefDetailImport.REFDETAIL_NAMESPACE;
682 cdmClass = Reference.class;
683 idSet = refDetailIdSet;
684 Map<String, Reference> refDetailMap= (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
685 result.put(nameSpace, refDetailMap);
686
687 } catch (SQLException e) {
688 throw new RuntimeException(e);
689 }
690 return result;
691 }
692
693
694 /**
695 * @param state
696 * @param media
697 * @param media
698 * @param descriptionSet
699 * @throws URISyntaxException
700 *
701 */
702 private TaxonDescription makeImage(BerlinModelImportState state, String fact, Media media, Set<TaxonDescription> descriptionSet, Taxon taxon) {
703 TaxonDescription taxonDescription = null;
704 try {
705 Reference sourceRef = state.getTransactionalSourceReference();
706 URI uri;
707 URI thumbUri;
708 if (state.getConfig().isSalvador()){
709 String thumbs = "thumbs/";
710 String uriStrFormat = "http://media.e-taxonomy.eu/salvador/berendsohn-et-al-%s/%s.jpg";
711 Integer intFact = Integer.valueOf(fact);
712 String vol = "2009";
713 int page = intFact + 249;
714 if (intFact >= 263){
715 vol = "2016";
716 page = intFact + (intFact < 403 ? 95 : 96);
717 }else if (intFact >= 142){
718 vol = "2012";
719 page = intFact + (intFact < 255 ? 3 : 4);
720 }
721 String title = getSalvadorImageTitle(intFact);
722 media.putTitle(Language.SPANISH_CASTILIAN(), title);
723
724 Reference ref = getSalvadorReference(vol);
725 String originalName = getSalvadorImageNameInfo(intFact);
726 IdentifiableSource source = media.addSource(OriginalSourceType.PrimaryMediaSource, fact, "Fig.", ref, String.valueOf(page));
727 source.setOriginalNameString(originalName);
728 media.setArtist(getSalvadorArtist());
729 media.addRights(getSalvadorCopyright(vol));
730 String uriStr = String.format(uriStrFormat, vol, fact);
731 String thumbUriStr = String.format(uriStrFormat, vol, thumbs + fact);
732 uri = new URI(uriStr);
733 thumbUri = new URI(thumbUriStr);
734 }else{
735 uri = new URI(fact.trim());
736 thumbUri = null;
737 }
738
739 makeMediaRepresentation(media, uri);
740 if (thumbUri != null){
741 makeMediaRepresentation(media, thumbUri);
742 }
743
744 taxonDescription = taxon.getOrCreateImageGallery(sourceRef == null ? null :sourceRef.getTitleCache());
745 } catch (URISyntaxException e) {
746 logger.warn("URISyntaxException. Image could not be imported: " + fact);
747 return null;
748 }
749 return taxonDescription;
750 }
751
752
753 private Map<Integer, String[]> salvadorImages = null;
754 private String getSalvadorImageTitle(Integer intFact) {
755 initSalvadorImagesFile();
756 String[] line = salvadorImages.get(intFact);
757 if (line == null){
758 logger.warn("Could not find salvador image metadata for " + intFact);
759 return String.valueOf(intFact);
760 }else{
761 int i = 2;
762 String result = CdmUtils.concat(" " + UTF8.EN_DASH + " ", line[i], line[i + 1]);
763 return result;
764 }
765 }
766
767 private String getSalvadorImageNameInfo(Integer intFact) {
768 initSalvadorImagesFile();
769 String[] line = salvadorImages.get(intFact);
770 if (line == null){
771 logger.warn("Could not find salvador image metadata for " + intFact);
772 return String.valueOf(intFact);
773 }else{
774 int i = 1;
775 String result = line[i].substring("Fig. ".length() + line[0].length()).trim();
776 return result;
777 }
778 }
779
780 /**
781 *
782 */
783 private void initSalvadorImagesFile() {
784 if (salvadorImages == null){
785 salvadorImages = new HashMap<>();
786 try {
787 CSVReader reader = new CSVReader(CdmUtils.getUtf8ResourceReader("salvador" + CdmUtils.getFolderSeperator() + "SalvadorImages.csv"),';');
788 List<String[]> lines = reader.readAll();
789 for (String[] line : lines){
790 if(! "ID".equals(line[0])){
791 salvadorImages.put(Integer.valueOf(line[0]), line);
792 }
793 }
794 reader.close();
795 } catch (IOException e) {
796 e.printStackTrace();
797 }
798 }
799 }
800
801
802 private Rights getSalvadorCopyright(String vol) {
803 String text;
804 if ("2009".equals(vol)){
805 text = "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2009.";
806 }else if ("2012".equals(vol)){
807 text = "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2012.";
808 }else if ("2016".equals(vol)){
809 text = "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2016.";
810 }else{
811 throw new RuntimeException("Volume not recognized: " + vol);
812 }
813 Rights result = Rights.NewInstance(text, Language.SPANISH_CASTILIAN(), RightsType.COPYRIGHT());
814 return result;
815 }
816
817
818 private Integer salvadorArtistId;
819 private AgentBase<?> getSalvadorArtist() {
820 if (salvadorArtistId == null){
821 Person person = Person.NewInstance();
822 person.setFirstname("José Gerver");
823 person.setLastname("Molina");
824 salvadorArtistId = getAgentService().save(person).getId();
825 return person;
826 }else{
827 return getAgentService().find(salvadorArtistId);
828 }
829 }
830
831 private Integer salvadorRef1Id;
832 private Integer salvadorRef2Id;
833 private Integer salvadorRef3Id;
834
835 private Reference getSalvadorReference(String vol){
836 if (salvadorRef1Id == null){
837 makeSalvadorReferences();
838 }
839 if ("2009".equals(vol)){
840 return getReferenceService().find(salvadorRef1Id);
841 }else if ("2012".equals(vol)){
842 return getReferenceService().find(salvadorRef2Id);
843 }else if ("2016".equals(vol)){
844 return getReferenceService().find(salvadorRef3Id);
845 }else{
846 throw new RuntimeException("Volume not recognized: " + vol);
847 }
848
849 }
850
851 private void makeSalvadorReferences() {
852 Person walter = Person.NewTitledInstance("Berendsohn, W. G.");
853 walter.setFirstname("Walter G.");
854 walter.setLastname("Berendsohn");
855 Person katja = Person.NewTitledInstance("Gruber, Anne Kathrina");
856 katja.setFirstname("Anne Katharina");
857 katja.setLastname("Gruber");
858 Person monte = Person.NewTitledInstance("Monterrosa Salomón, J.");
859 Person olmedo = Person.NewTitledInstance("Olmedo Galán, P.");
860 Person rodriguez = Person.NewTitledInstance("Rodríguez Delcid, D");
861
862 Team team1 = Team.NewInstance();
863 team1.addTeamMember(walter);
864 team1.addTeamMember(katja);
865 team1.addTeamMember(monte);
866
867 Team team2 = Team.NewInstance();
868 team2.addTeamMember(walter);
869 team2.addTeamMember(katja);
870 team2.addTeamMember(rodriguez);
871 team2.addTeamMember(olmedo);
872
873 Reference vol1 = ReferenceFactory.newBook();
874 Reference vol2 = ReferenceFactory.newBook();
875 Reference vol3 = ReferenceFactory.newBook();
876
877 vol1.setAuthorship(team1);
878 vol2.setAuthorship(team1);
879 vol3.setAuthorship(team2);
880
881 vol1.setDatePublished(TimePeriodParser.parseString("2009"));
882 vol2.setDatePublished(TimePeriodParser.parseString("2012"));
883 vol3.setDatePublished(TimePeriodParser.parseString("2016"));
884
885 Reference englera = ReferenceFactory.newPrintSeries("Englera");
886 vol1.setInSeries(englera);
887 vol2.setInSeries(englera);
888 vol3.setInSeries(englera);
889
890 vol1.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 1: Angiospermae - Familias A-L");
891 vol2.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 2: Angiospermae - Familias M-P y Pteridophyta");
892 vol3.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 3: Angiospermae - Familias R-Z y Gymnospermae");
893
894 vol1.setVolume("29(1)");
895 vol2.setVolume("29(2)");
896 vol3.setVolume("29(3)");
897
898 vol1.setPages("1-438");
899 vol2.setVolume("1-300");
900 vol3.setVolume("1-356");
901
902 salvadorRef1Id = getReferenceService().save(vol1).getId();
903 salvadorRef2Id = getReferenceService().find(getReferenceService().saveOrUpdate(vol2)).getId();
904 salvadorRef3Id = getReferenceService().find(getReferenceService().saveOrUpdate(vol3)).getId();
905 return;
906 }
907
908
909 /**
910 * @param media
911 * @param uri
912 * @param imageInfo
913 * @param size
914 */
915 private void makeMediaRepresentation(Media media, URI uri) {
916 ImageInfo imageInfo = null;
917 Integer size = null;
918 try {
919 imageInfo = ImageInfo.NewInstance(uri, 0);
920 } catch (IOException | HttpException e) {
921 logger.error("Error when reading image meta: " + e + ", "+ uri.toString());
922 }
923 String mimeType = imageInfo == null ? null : imageInfo.getMimeType();
924 String suffix = imageInfo == null ? null : imageInfo.getSuffix();
925 MediaRepresentation mediaRepresentation = MediaRepresentation.NewInstance(mimeType, suffix);
926 media.addRepresentation(mediaRepresentation);
927 ImageFile image = ImageFile.NewInstance(uri, size, imageInfo);
928 mediaRepresentation.addRepresentationPart(image);
929 }
930
931 private TaxonBase<?> getTaxon(Map<String, TaxonBase> taxonMap, Integer taxonIdObj, Number taxonId){
932 if (taxonIdObj != null){
933 return taxonMap.get(String.valueOf(taxonId));
934 }else{
935 return null;
936 }
937
938 }
939
940 private Feature getFeature(Map<Integer, Feature> featureMap, Integer categoryFkInt){
941 if (categoryFkInt != null){
942 return featureMap.get(categoryFkInt);
943 }else{
944 return null;
945 }
946
947 }
948
949 @Override
950 protected boolean doCheck(BerlinModelImportState state){
951 IOValidator<BerlinModelImportState> validator = new BerlinModelFactsImportValidator();
952 return validator.validate(state);
953 }
954
955 @Override
956 protected boolean isIgnore(BerlinModelImportState state){
957 return ! state.getConfig().isDoFacts();
958 }
959
960
961
962 }