BGBM BDI Projects / cdmlib-apps.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
ref #8257 remove factory method for term node creation in app-import
[cdmlib-apps.git] / app-import / src / main / java / eu / etaxonomy / cdm / io / berlinModel / in / BerlinModelFactsImport.java
1 /**
2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
5 *
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
8 */
9
10 package eu.etaxonomy.cdm.io.berlinModel.in;
11
12 import java.io.IOException;
13 import java.net.URI;
14 import java.net.URISyntaxException;
15 import java.sql.ResultSet;
16 import java.sql.SQLException;
17 import java.util.HashMap;
18 import java.util.HashSet;
19 import java.util.List;
20 import java.util.Map;
21 import java.util.Set;
22 import java.util.UUID;
23
24 import javax.validation.constraints.NotNull;
25
26 import org.apache.commons.lang.StringUtils;
27 import org.apache.http.HttpException;
28 import org.apache.log4j.Logger;
29 import org.springframework.stereotype.Component;
30
31 import au.com.bytecode.opencsv.CSVReader;
32 import eu.etaxonomy.cdm.common.CdmUtils;
33 import eu.etaxonomy.cdm.common.UTF8;
34 import eu.etaxonomy.cdm.common.media.ImageInfo;
35 import eu.etaxonomy.cdm.database.update.DatabaseTypeNotSupportedException;
36 import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
37 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelFactsImportValidator;
38 import eu.etaxonomy.cdm.io.common.IOValidator;
39 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
40 import eu.etaxonomy.cdm.io.common.Source;
41 import eu.etaxonomy.cdm.model.agent.AgentBase;
42 import eu.etaxonomy.cdm.model.agent.Person;
43 import eu.etaxonomy.cdm.model.agent.Team;
44 import eu.etaxonomy.cdm.model.common.Annotation;
45 import eu.etaxonomy.cdm.model.common.AnnotationType;
46 import eu.etaxonomy.cdm.model.common.CdmBase;
47 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
48 import eu.etaxonomy.cdm.model.common.Language;
49 import eu.etaxonomy.cdm.model.common.LanguageString;
50 import eu.etaxonomy.cdm.model.common.Marker;
51 import eu.etaxonomy.cdm.model.common.MarkerType;
52 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
53 import eu.etaxonomy.cdm.model.description.DescriptionElementBase;
54 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
55 import eu.etaxonomy.cdm.model.description.Distribution;
56 import eu.etaxonomy.cdm.model.description.Feature;
57 import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
58 import eu.etaxonomy.cdm.model.description.TaxonDescription;
59 import eu.etaxonomy.cdm.model.description.TextData;
60 import eu.etaxonomy.cdm.model.location.NamedArea;
61 import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
62 import eu.etaxonomy.cdm.model.location.NamedAreaType;
63 import eu.etaxonomy.cdm.model.media.ImageFile;
64 import eu.etaxonomy.cdm.model.media.Media;
65 import eu.etaxonomy.cdm.model.media.MediaRepresentation;
66 import eu.etaxonomy.cdm.model.media.Rights;
67 import eu.etaxonomy.cdm.model.media.RightsType;
68 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
69 import eu.etaxonomy.cdm.model.reference.Reference;
70 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
71 import eu.etaxonomy.cdm.model.taxon.Taxon;
72 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
73 import eu.etaxonomy.cdm.model.term.FeatureTree;
74 import eu.etaxonomy.cdm.model.term.Representation;
75 import eu.etaxonomy.cdm.model.term.TermBase;
76 import eu.etaxonomy.cdm.model.term.TermType;
77 import eu.etaxonomy.cdm.model.term.TermVocabulary;
78 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
79 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
80
81 /**
82 * @author a.mueller
83 * @since 20.03.2008
84 */
85 @Component
86 public class BerlinModelFactsImport extends BerlinModelImportBase {
87 private static final long serialVersionUID = 4095154818163504795L;
88
89 private static final Logger logger = Logger.getLogger(BerlinModelFactsImport.class);
90
91 public static final String NAMESPACE = "Fact";
92
93 public static final String SEQUENCE_PREFIX = "ORDER: ";
94
95 private int modCount = 10000;
96 private static final String pluralString = "facts";
97 private static final String dbTableName = "Fact";
98
99 //FIXME don't use as class variable
100 private Map<Integer, Feature> featureMap;
101
102 public BerlinModelFactsImport(){
103 super(dbTableName, pluralString);
104 }
105
106
107 private TermVocabulary<Feature> getFeatureVocabulary(){
108 TermVocabulary<Feature> newVoc = TermVocabulary.NewInstance(TermType.Feature, "Berlin Model Import Feature Vocabulary", "Berlin Model Import Feature Vocabulary", null, null);
109 getVocabularyService().save(newVoc);
110
111 return newVoc;
112 }
113
114 private Map<Integer, Feature> invokeFactCategories(BerlinModelImportState state){
115
116 Map<Integer, Feature> result = state.getConfig().getFeatureMap();
117 Source source = state.getConfig().getSource();
118 boolean createFeatureTree = state.getConfig().isSalvador(); //for some reason feature tree creation does not work for salavdor
119
120 FeatureTree featureTree = (!createFeatureTree) ? null : FeatureTree.NewInstance(state.getConfig().getFeatureTreeUuid());
121 if (featureTree!= null && createFeatureTree){
122 featureTree.setTitleCache(state.getConfig().getFeatureTreeTitle(), true);
123 }
124
125 try {
126 //get data from database
127 String strQuery =
128 " SELECT FactCategory.* " +
129 " FROM FactCategory "+
130 " WHERE (1=1)";
131 if (state.getConfig().isSalvador()){
132 strQuery += " AND " + state.getConfig().getFactFilter().replace("factCategoryFk", "factCategoryId");
133 }
134
135 ResultSet rs = source.getResultSet(strQuery) ;
136
137 TermVocabulary<Feature> featureVocabulary = getFeatureVocabulary();
138 int i = 0;
139 //for each reference
140 while (rs.next()){
141
142 if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("FactCategories handled: " + (i-1));}
143
144 int factCategoryId = rs.getInt("factCategoryId");
145 String factCategory = rs.getString("factCategory");
146
147 Feature feature;
148 try {
149 feature = BerlinModelTransformer.factCategory2Feature(factCategoryId);
150 } catch (UnknownCdmTypeException e) {
151 UUID featureUuid = null;
152 featureUuid = BerlinModelTransformer.getFeatureUuid(String.valueOf(factCategoryId+"-"+factCategory));
153 if (featureUuid == null){
154 logger.warn("New Feature (FactCategoryId: " + factCategoryId + ")");
155 featureUuid = UUID.randomUUID();
156 }
157 feature = getFeature(state, featureUuid, factCategory, factCategory, null, featureVocabulary);
158 if (state.getConfig().isSalvador()){
159 adaptNewSalvadorFeature(factCategoryId, feature);
160 }
161 //id
162 doId(state, feature, factCategoryId, "FactCategory");
163
164 //TODO
165 // MaxFactNumber int Checked
166 // ExtensionTableName varchar(100) Checked
167 // Description nvarchar(1000) Checked
168 // locExtensionFormName nvarchar(80) Checked
169 // RankRestrictionFk int Checked
170 }
171
172 result.put(factCategoryId, feature);
173 if (createFeatureTree && isPublicFeature(factCategoryId)){
174 featureTree.getRoot().addChild(feature);
175 }
176 }
177 if (createFeatureTree){
178 featureTree.getRoot().addChild(Feature.DISTRIBUTION(),2);
179 featureTree.getRoot().addChild(Feature.NOTES(), featureTree.getRoot().getChildCount()-1);
180 getFeatureTreeService().save(featureTree);
181 }
182 return result;
183 } catch (SQLException e) {
184 logger.error("SQLException:" + e);
185 return null;
186 }
187 }
188
189
190 /**
191 * @param factCategoryId
192 * @param feature
193 */
194 private void adaptNewSalvadorFeature(int factCategoryId, Feature feature) {
195 if (factCategoryId == 306){
196 addSpanishRepresentationLabel(feature, "Nombre(s) común(es)");
197 } else if (factCategoryId == 307){
198 addSpanishRepresentationLabel(feature, "Muestras de herbario");
199 } else if (factCategoryId == 310){
200 addEnglishFactCategoryName(feature, "Other references for taxon");
201 } else if (factCategoryId == 309){
202 addEnglishFactCategoryName(feature, "Report (reference) for El Salvador");
203 } else if (factCategoryId == 311){
204 addEnglishFactCategoryName(feature, "Taxon illustration references");
205 } else if (factCategoryId == 312){
206 addSpanishRepresentationLabel(feature, "Imágen");
207 } else if (factCategoryId == 350){
208 addSpanishRepresentationLabel(feature, "Descripción");
209 } else if (factCategoryId == 303){
210 addEnglishFactCategoryName(feature, "General distribution");
211 } else if (factCategoryId == 2000){
212 addEnglishFactCategoryName(feature, "Habitat in El Salvador");
213 } else if (factCategoryId == 302){
214 addSpanishRepresentationLabel(feature, "Usos");
215 } else if (factCategoryId == 1800){
216 addEnglishFactCategoryName(feature, "Specimen notes");
217 } else if (factCategoryId == 1900){
218 addEnglishFactCategoryName(feature, "Editorial notes");
219 }
220
221 }
222
223
224 /**
225 * @param feature
226 * @param string
227 */
228 private void addSpanishRepresentationLabel(TermBase term, String label) {
229 term.getRepresentations().add(Representation.NewInstance(label, label, null, Language.SPANISH_CASTILIAN()));
230 }
231
232 /**
233 * @param feature
234 * @param string
235 */
236 private void addEnglishFactCategoryName(Feature feature, String label) {
237 feature.getRepresentations().iterator().next().setLanguage(Language.SPANISH_CASTILIAN());
238 feature.getRepresentations().add(Representation.NewInstance(label, label, null, Language.ENGLISH()));
239 }
240
241
242 @Override
243 protected void doInvoke(BerlinModelImportState state) {
244 if (state.getConfig().isSalvador()){
245 invokeSpanishTermLabels();
246 }
247 featureMap = invokeFactCategories(state);
248 super.doInvoke(state);
249 return;
250 }
251
252
253 /**
254 *
255 */
256 private void invokeSpanishTermLabels() {
257 addSpanishRepresentationLabel(Feature.NOTES(), "Notas");
258 addSpanishRepresentationLabel(NamedAreaLevel.DEPARTMENT(), "Departamento");
259 addSpanishRepresentationLabel(PresenceAbsenceTerm.NATIVE(), "nativo");
260 addSpanishRepresentationLabel(PresenceAbsenceTerm.CULTIVATED(), "cultivado");
261 addSpanishRepresentationLabel(PresenceAbsenceTerm.PRESENT(), "presente");
262
263
264 }
265
266
267 @Override
268 protected String getIdQuery(BerlinModelImportState state) {
269 String result = super.getIdQuery(state);
270 if (StringUtils.isNotBlank(state.getConfig().getFactFilter())){
271 result += " WHERE " + state.getConfig().getFactFilter();
272 }else{
273 result = super.getIdQuery(state);
274 }
275 result += getOrderBy(state.getConfig());
276 return result;
277 }
278
279 @Override
280 protected String getRecordQuery(BerlinModelImportConfigurator config) {
281 String strQuery =
282 " SELECT Fact.*, PTaxon.RIdentifier as taxonId, RefDetail.Details " +
283 " FROM Fact " +
284 " INNER JOIN PTaxon ON Fact.PTNameFk = PTaxon.PTNameFk AND Fact.PTRefFk = PTaxon.PTRefFk " +
285 " LEFT OUTER JOIN RefDetail ON Fact.FactRefDetailFk = RefDetail.RefDetailId AND Fact.FactRefFk = RefDetail.RefFk " +
286 " WHERE (FactId IN (" + ID_LIST_TOKEN + "))";
287 strQuery += getOrderBy(config);
288
289 return strQuery;
290 }
291
292
293 private String getOrderBy(BerlinModelImportConfigurator config) {
294 String result;
295 try{
296 if (config.getSource().checkColumnExists("Fact", "Sequence")){
297 result = " ORDER By Fact.Sequence, Fact.FactId";
298 if (config.isSalvador()){
299 result = " ORDER By Fact.FactCategoryFk, Fact.Sequence, Fact.FactId";
300 }
301 }else{
302 result = " ORDER By Fact.FactId";
303 }
304 } catch (DatabaseTypeNotSupportedException e) {
305 logger.info("checkColumnExists not supported");
306 result = " ORDER By Fact.FactId";
307 }
308 return result;
309 }
310
311 @Override
312 public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, BerlinModelImportState state) {
313 boolean success = true ;
314 Set<TaxonBase> taxaToSave = new HashSet<>();
315 Map<String, TaxonBase> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
316 Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
317
318 ResultSet rs = partitioner.getResultSet();
319
320 Reference sourceRef = state.getTransactionalSourceReference();
321
322 try{
323 int i = 0;
324 //for each fact
325 while (rs.next()){
326 try{
327 if ((i++ % modCount) == 0){ logger.info("Facts handled: " + (i-1));}
328
329 int factId = rs.getInt("factId");
330 Integer taxonId = nullSafeInt(rs, "taxonId");
331 Integer factRefFkInt = nullSafeInt(rs, "factRefFk");
332 Integer categoryFkInt = nullSafeInt(rs, "factCategoryFk");
333 String details = rs.getString("Details");
334 String fact = CdmUtils.Nz(rs.getString("Fact"));
335 String notes = CdmUtils.Nz(rs.getString("notes"));
336 Boolean doubtfulFlag = rs.getBoolean("DoubtfulFlag");
337
338 TaxonBase<?> taxonBase = getTaxon(taxonMap, taxonId, taxonId);
339 Feature feature = getFeature(featureMap, categoryFkInt) ;
340
341 if (taxonBase == null){
342 logger.warn("Taxon for Fact " + factId + " does not exist in store");
343 success = false;
344 }else{
345 TaxonDescription taxonDescription;
346 if ( (taxonDescription = getMyTaxonDescripion(taxonBase, state, categoryFkInt, taxonId, factId, fact, sourceRef)) == null){
347 success = false;
348 continue;
349 }
350
351 //textData
352 TextData textData = null;
353 boolean newTextData = true;
354
355 // For Cichorieae DB: If fact category is 31 (Systematics) and there is already a Systematics TextData
356 // description element append the fact text to the existing TextData
357 if(categoryFkInt.equals(31)) {
358 Set<DescriptionElementBase> descriptionElements = taxonDescription.getElements();
359 for (DescriptionElementBase descriptionElement : descriptionElements) {
360 String featureString = descriptionElement.getFeature().getRepresentation(Language.DEFAULT()).getLabel();
361 if (descriptionElement instanceof TextData && featureString.equals("Systematics")) { // TODO: test
362 textData = (TextData)descriptionElement;
363 String factTextStr = textData.getText(Language.DEFAULT());
364 // FIXME: Removing newlines doesn't work
365 if (factTextStr.contains("\\r\\n")) {
366 factTextStr = factTextStr.replaceAll("\\r\\n","");
367 }
368 StringBuilder factText = new StringBuilder(factTextStr);
369 factText.append(fact);
370 fact = factText.toString();
371 newTextData = false;
372 break;
373 }
374 }
375 }
376
377 if (taxonDescription.isImageGallery()){
378 newTextData = false;
379 textData = (TextData)taxonDescription.getElements().iterator().next();
380 }
381 if(newTextData == true) {
382 textData = TextData.NewInstance();
383 }else if (textData == null){
384 throw new RuntimeException("Textdata must not be null"); //does not happen, only to avoid potential NPE warnings
385 }
386
387
388 //for diptera database
389 if (categoryFkInt.equals(99) && notes.contains("<OriginalName>")){
390 fact = notes + ": " + fact ;
391 }
392 //for E+M maps
393 if (categoryFkInt.equals(14) && state.getConfig().isRemoveHttpMapsAnchor() && fact.contains("<a href")){
394 //example <a href="http://euromed.luomus.fi/euromed_map.php?taxon=280629&size=medium">distribution</a>
395 fact = fact.replace("<a href=\"", "").replace("\">distribution</a>", "");
396 }
397
398 //TODO textData.putText(fact, bmiConfig.getFactLanguage()); //doesn't work because bmiConfig.getFactLanguage() is not not a persistent Language Object
399 //throws in thread "main" org.springframework.dao.InvalidDataAccessApiUsageException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language; nested exception is org.hibernate.TransientObjectException: object references an unsaved transient instance - save the transient instance before flushing: eu.etaxonomy.cdm.model.common.Language
400
401 Language lang = Language.DEFAULT();
402 if (state.getConfig().isSalvador()){
403 lang = getSalvadorFactLanguage(categoryFkInt);
404 }
405 if (! taxonDescription.isImageGallery()){
406 textData.putText(lang, fact);
407 textData.setFeature(feature);
408 }
409
410 DescriptionElementBase deb = textData;
411
412 if (state.getConfig().isSalvador()){
413 if (categoryFkInt == 306){
414 NamedArea area = null; // for now we do not set an area as it can not be disabled in dataportals via css yet
415 deb = CommonTaxonName.NewInstance(fact, Language.SPANISH_CASTILIAN(), area);
416 }else if (categoryFkInt == 307){
417 Distribution salvadorDistribution = salvadorDistributionFromMuestrasDeHerbar((Taxon)taxonBase, fact);
418 if (salvadorDistribution != null){
419 //id
420 doId(state, salvadorDistribution, factId, "Fact");
421 mergeSalvadorDistribution(taxonDescription, salvadorDistribution);
422 }
423 }
424 }
425
426 //reference
427 Reference citation = null;
428 String factRefFk = String.valueOf(factRefFkInt);
429 if (factRefFkInt != null){
430 citation = refMap.get(factRefFk);
431 }
432 if (citation == null && (factRefFkInt != null)){
433 logger.warn("Citation not found in referenceMap: " + factRefFk);
434 success = false;
435 }
436 if (citation != null || StringUtils.isNotBlank(details)){
437 DescriptionElementSource originalSource = DescriptionElementSource.NewPrimarySourceInstance(citation, details);
438 deb.addSource(originalSource);
439 }
440 taxonDescription.addElement(deb);
441 //doubtfulFlag
442 if (doubtfulFlag){
443 deb.addMarker(Marker.NewInstance(MarkerType.IS_DOUBTFUL(), true));
444 }
445 //publisheFlag
446 String strPublishFlag = "publishFlag";
447 boolean publishFlagExists = state.getConfig().getSource().checkColumnExists(dbTableName, strPublishFlag);
448 if (publishFlagExists){
449 Boolean publishFlag = rs.getBoolean(strPublishFlag);
450 if (publishFlag == false){
451 deb.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), publishFlag));
452 }
453 }
454
455 //Sequence
456 Integer sequence = rs.getInt("Sequence");
457 if (sequence != 999){
458 String strSequence = String.valueOf(sequence);
459 strSequence = SEQUENCE_PREFIX + strSequence;
460 //TODO make it an Extension when possible
461 //Extension datesExtension = Extension.NewInstance(textData, strSequence, ExtensionType.ORDER());
462 Annotation annotation = Annotation.NewInstance(strSequence, AnnotationType.TECHNICAL(), Language.ENGLISH());
463 deb.addAnnotation(annotation);
464 }
465
466 // if (categoryFkObj == FACT_DESCRIPTION){
467 // //;
468 // }else if (categoryFkObj == FACT_OBSERVATION){
469 // //;
470 // }else if (categoryFkObj == FACT_DISTRIBUTION_EM){
471 // //
472 // }else {
473 // //TODO
474 // //logger.warn("FactCategory " + categoryFk + " not yet implemented");
475 // }
476
477 //notes
478 doCreatedUpdatedNotes(state, deb, rs);
479 doId(state, deb, factId, "Fact");
480
481 //TODO
482 //Designation References -> unclear how to map to CDM
483
484
485 //sequence -> textData is not an identifiable entity therefore extensions are not possible
486 //fact category better
487
488 taxaToSave.add(taxonBase);
489 }
490 } catch (Exception re){
491 logger.error("An exception occurred during the facts import");
492 re.printStackTrace();
493 success = false;
494 }
495 //put
496 }
497 logger.info("Facts handled: " + (i-1));
498 logger.info("Taxa to save: " + taxaToSave.size());
499 getTaxonService().save(taxaToSave);
500 }catch(SQLException e){
501 throw new RuntimeException(e);
502 }
503 return success;
504 }
505
506
507 /**
508 * @param taxonDescription
509 * @param salvadorDistribution
510 */
511 private void mergeSalvadorDistribution(TaxonDescription taxonDescription,
512 @NotNull Distribution newDistribution) {
513
514 Distribution existingDistribution = null;
515 for (DescriptionElementBase deb : taxonDescription.getElements()){
516 if (deb.isInstanceOf(Distribution.class)){
517 Distribution distribution = CdmBase.deproxy(deb, Distribution.class);
518 if (distribution.getArea() != null && distribution.getArea().equals(newDistribution.getArea())){
519 existingDistribution = distribution;
520 break;
521 }else if (distribution.getArea() == null){
522 logger.warn("Area for distribution is null: " + distribution.getUuid());
523 }
524 }
525 }
526 if (existingDistribution == null){
527 taxonDescription.addElement(newDistribution);
528 }else if(!existingDistribution.getStatus().equals(newDistribution.getStatus())){
529 //should not happen
530 logger.warn("Taxon has areas with different distribution states: " + taxonDescription.getTaxon().getTitleCache());
531 }else{
532 //do nothing, distribution already exists
533 }
534 }
535
536
537 private Map<String, NamedArea> salvadorAreaMap = null;
538 private Distribution salvadorDistributionFromMuestrasDeHerbar(Taxon taxon, String fact) {
539 if (salvadorAreaMap == null){
540 salvadorAreaMap = new HashMap<>();
541 TermVocabulary<NamedArea> salvadorAreas = getVocabulary(TermType.NamedArea, BerlinModelTransformer.uuidSalvadorAreas,
542 "Salvador areas", "Salvador areas", null, null, true, NamedArea.NewInstance());
543 getVocabularyService().save(salvadorAreas);
544 }
545 Distribution result = null;
546 String[] areaStrings = fact.split(":");
547 if (areaStrings.length > 1){
548 String areaString = areaStrings[0];
549 NamedArea area = salvadorAreaMap.get(areaString);
550 if (area == null){
551 logger.info("Added Salvador area: " + areaString);
552 TermVocabulary<NamedArea> voc = getVocabulary(TermType.NamedArea, BerlinModelTransformer.uuidSalvadorAreas,
553 "Salvador departments", "Salvador departments", null, null, true, NamedArea.NewInstance());
554 if (voc.getRepresentation(Language.SPANISH_CASTILIAN()) == null){
555 voc.addRepresentation(Representation.NewInstance("Salvador departamentos", "Salvador departamentos", "dep.", Language.SPANISH_CASTILIAN()));
556 getVocabularyService().saveOrUpdate(voc);
557 }
558 NamedArea newArea = NamedArea.NewInstance(areaString, areaString, null);
559 newArea.getRepresentations().iterator().next().setLanguage(Language.SPANISH_CASTILIAN());
560 newArea.setLevel(NamedAreaLevel.DEPARTMENT());
561 newArea.setType(NamedAreaType.ADMINISTRATION_AREA());
562 voc.addTerm(newArea);
563 getTermService().saveOrUpdate(newArea);
564 salvadorAreaMap.put(areaString, newArea);
565 area = newArea;
566 }
567 PresenceAbsenceTerm state = getSalvadorDistributionState(taxon);
568 result = Distribution.NewInstance(area, state);
569 return result;
570 }else{
571 return null;
572 }
573 }
574
575 private PresenceAbsenceTerm getSalvadorDistributionState(Taxon taxon) {
576 boolean hasGlobalDist = false;
577 for (TaxonDescription desc : taxon.getDescriptions()){
578 for (DescriptionElementBase deb : desc.getElements()){
579 if (deb.getFeature().getUuid().equals(BerlinModelTransformer.uuidFeatureDistributionGlobal)){
580 hasGlobalDist = true;
581 TextData textData = CdmBase.deproxy(deb, TextData.class);
582 for (LanguageString text : textData.getMultilanguageText().values()){
583 if (text.getText().contains("El Salvador")){
584 return PresenceAbsenceTerm.NATIVE();
585 }
586 }
587 }
588 }
589 }
590 if (!hasGlobalDist){
591 logger.warn("No global distribution found: " + taxon.getTitleCache());
592 }
593 return hasGlobalDist ? PresenceAbsenceTerm.CULTIVATED(): PresenceAbsenceTerm.PRESENT();
594 }
595
596
597 /**
598 * @param factId
599 * @return
600 */
601 private Language getSalvadorFactLanguage(int categoryFkInt) {
602 if (categoryFkInt == 350){
603 return Language.ENGLISH();
604 }else if (categoryFkInt == 1800 || categoryFkInt == 1900){
605 return Language.UNDETERMINED();
606 }
607 return Language.SPANISH_CASTILIAN();
608 }
609
610
611 private TaxonDescription getMyTaxonDescripion(TaxonBase taxonBase, BerlinModelImportState state, Integer categoryFk, Integer taxonId, int factId, String fact, Reference sourceRef) {
612 Taxon taxon = null;
613 if ( taxonBase instanceof Taxon ) {
614 taxon = (Taxon) taxonBase;
615 }else{
616 logger.warn("TaxonBase " + (taxonId==null?"(null)":taxonId) + " for Fact " + factId + " was not of type Taxon but: " + taxonBase.getClass().getSimpleName());
617 return null;
618 }
619
620 TaxonDescription taxonDescription = null;
621 Set<TaxonDescription> descriptionSet= taxon.getDescriptions();
622
623 Media media = null;
624 //for diptera / salvador images
625 if (categoryFk == 51 || categoryFk == 312){ //TODO check also FactCategory string
626 media = Media.NewInstance();
627 taxonDescription = makeImage(state, fact, media, descriptionSet, taxon);
628
629 if (taxonDescription == null){
630 return null;
631 }
632
633 TextData textData = null;
634 for (DescriptionElementBase el: taxonDescription.getElements()){
635 if (el.isInstanceOf(TextData.class)){
636 textData = CdmBase.deproxy(el, TextData.class);
637 }
638 }
639 if (textData == null){
640 textData = TextData.NewInstance(Feature.IMAGE());
641 taxonDescription.addElement(textData);
642 }
643 textData.addMedia(media);
644 }
645 //all others (no image) -> getDescription
646 else{
647 boolean isPublic = isPublicFeature(categoryFk);
648 for (TaxonDescription desc: descriptionSet){
649
650 if (! desc.isImageGallery()){
651 if (state.getConfig().isSalvador()){
652 if (desc.isDefault() && isPublic || !desc.isDefault() && !isPublic){
653 taxonDescription = desc;
654 break;
655 }
656 }else{
657 taxonDescription = desc;
658 break;
659 }
660 }
661 }
662 if (taxonDescription == null){
663 taxonDescription = TaxonDescription.NewInstance();
664 if (state.getConfig().isSalvador()){
665 String title = "Factual data for " + taxon.getName().getTitleCache();
666 if (isPublic){
667 taxonDescription.setDefault(isPublic);
668 }else{
669 title = "Non public f" + title.substring(1);
670 }
671 taxonDescription.setTitleCache(title, true);
672 }else if(state.getConfig().isEuroMed()){
673 taxonDescription.setDefault(true);
674 }else{
675 taxonDescription.setTitleCache(sourceRef == null ? null : sourceRef.getTitleCache(), true);
676 }
677 taxon.addDescription(taxonDescription);
678 }
679 }
680 return taxonDescription;
681 }
682
683
684 /**
685 * @param categoryFk
686 * @return
687 */
688 private boolean isPublicFeature(Integer categoryFk) {
689 return ! (categoryFk == 1800 || categoryFk == 1900 || categoryFk == 2000);
690 }
691
692
693 @Override
694 public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
695 String nameSpace;
696 Class<?> cdmClass;
697 Set<String> idSet;
698 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
699
700 try{
701 Set<String> taxonIdSet = new HashSet<String>();
702 Set<String> referenceIdSet = new HashSet<String>();
703 Set<String> refDetailIdSet = new HashSet<String>();
704 while (rs.next()){
705 handleForeignKey(rs, taxonIdSet, "taxonId");
706 handleForeignKey(rs, referenceIdSet, "FactRefFk");
707 handleForeignKey(rs, referenceIdSet, "PTDesignationRefFk");
708 handleForeignKey(rs, refDetailIdSet, "FactRefDetailFk");
709 handleForeignKey(rs, refDetailIdSet, "PTDesignationRefDetailFk");
710 }
711
712 //taxon map
713 nameSpace = BerlinModelTaxonImport.NAMESPACE;
714 cdmClass = TaxonBase.class;
715 idSet = taxonIdSet;
716 Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
717 result.put(nameSpace, taxonMap);
718
719 //reference map
720 nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
721 cdmClass = Reference.class;
722 idSet = referenceIdSet;
723 Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
724 result.put(nameSpace, referenceMap);
725
726 //refDetail map
727 nameSpace = BerlinModelRefDetailImport.REFDETAIL_NAMESPACE;
728 cdmClass = Reference.class;
729 idSet = refDetailIdSet;
730 Map<String, Reference> refDetailMap= (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
731 result.put(nameSpace, refDetailMap);
732
733 } catch (SQLException e) {
734 throw new RuntimeException(e);
735 }
736 return result;
737 }
738
739
740 /**
741 * @param state
742 * @param media
743 * @param media
744 * @param descriptionSet
745 * @throws URISyntaxException
746 *
747 */
748 private TaxonDescription makeImage(BerlinModelImportState state, String fact, Media media, Set<TaxonDescription> descriptionSet, Taxon taxon) {
749 TaxonDescription taxonDescription = null;
750 try {
751 Reference sourceRef = state.getTransactionalSourceReference();
752 URI uri;
753 URI thumbUri;
754 if (state.getConfig().isSalvador()){
755 String thumbs = "thumbs/";
756 String uriStrFormat = "http://media.e-taxonomy.eu/salvador/berendsohn-et-al-%s/%s.jpg";
757 Integer intFact = Integer.valueOf(fact);
758 String vol = "2009";
759 int page = intFact + 249;
760 if (intFact >= 263){
761 vol = "2016";
762 page = intFact - (intFact < 403 ? 95 : 94);
763 }else if (intFact >= 142){
764 vol = "2012";
765 page = intFact + (intFact < 255 ? 3 : 4);
766 }
767
768 String title = getSalvadorImageTitle(intFact, vol);
769 media.putTitle(Language.LATIN(), title);
770 String description = getSalvadorImageDescription(intFact);
771 media.putDescription(Language.SPANISH_CASTILIAN(), description);
772
773 Reference ref = getSalvadorReference(vol);
774 String originalName = getSalvadorImageNameInfo(intFact);
775 IdentifiableSource source = media.addSource(OriginalSourceType.PrimaryMediaSource, fact, "Fig.", ref, "p. " + page);
776 source.setOriginalNameString(originalName);
777 media.setArtist(getSalvadorArtist());
778 media.addRights(getSalvadorCopyright(vol));
779 String uriStr = String.format(uriStrFormat, vol, fact);
780 String thumbUriStr = String.format(uriStrFormat, vol, thumbs + fact);
781 uri = new URI(uriStr);
782 thumbUri = new URI(thumbUriStr);
783 }else{
784 uri = new URI(fact.trim());
785 thumbUri = null;
786 }
787
788 makeMediaRepresentation(media, uri);
789 if (thumbUri != null){
790 makeMediaRepresentation(media, thumbUri);
791 }
792
793 taxonDescription = taxon.getOrCreateImageGallery(sourceRef == null ? null :sourceRef.getTitleCache());
794 } catch (URISyntaxException e) {
795 logger.warn("URISyntaxException. Image could not be imported: " + fact);
796 return null;
797 }
798 return taxonDescription;
799 }
800
801
802 /**
803 * @param intFact
804 * @param vol
805 * @return
806 */
807 private String getSalvadorImageTitle(Integer intFact, String vol) {
808 initSalvadorImagesFile();
809 String[] line = salvadorImages.get(intFact);
810 if (line == null){
811 logger.warn("Could not find salvador image metadata for " + intFact);
812 return String.valueOf(intFact);
813 }else{
814 String name = getSalvadorImageNameInfo(intFact);
815 String result = UTF8.QUOT_DBL_LEFT + name + UTF8.QUOT_DBL_RIGHT + " [Berendsohn & al. " + vol + "]";
816 return result;
817 }
818 }
819
820
821 private Map<Integer, String[]> salvadorImages = null;
822 private String getSalvadorImageDescription(Integer intFact) {
823 initSalvadorImagesFile();
824 String[] line = salvadorImages.get(intFact);
825 if (line == null){
826 logger.warn("Could not find salvador image metadata for " + intFact);
827 return String.valueOf(intFact);
828 }else{
829 int i = 2;
830 String result = CdmUtils.concat(" " + UTF8.EN_DASH + " ", line[i], line[i + 1]);
831 return result;
832 }
833 }
834
835 private String getSalvadorImageNameInfo(Integer intFact) {
836 initSalvadorImagesFile();
837 String[] line = salvadorImages.get(intFact);
838 if (line == null){
839 logger.warn("Could not find salvador image metadata for " + intFact);
840 return String.valueOf(intFact);
841 }else{
842 int i = 1;
843 String result = line[i].substring("Fig. ".length() + line[0].length()).trim();
844 return result;
845 }
846 }
847
848 private void initSalvadorImagesFile() {
849 if (salvadorImages == null){
850 salvadorImages = new HashMap<>();
851 try {
852 CSVReader reader = new CSVReader(CdmUtils.getUtf8ResourceReader("salvador" + CdmUtils.getFolderSeperator() + "SalvadorImages.csv"),';');
853 List<String[]> lines = reader.readAll();
854 for (String[] line : lines){
855 String first = line[0];
856 if(! "ID".equals(first)){
857 try {
858 salvadorImages.put(Integer.valueOf(first), line);
859 } catch (NumberFormatException e) {
860 logger.warn("Number not recognized: " + first);
861 }
862 }
863 }
864 reader.close();
865 } catch (IOException e) {
866 e.printStackTrace();
867 }
868 }
869 }
870
871
872 private Rights rights1;
873 private Rights rights2;
874 private Rights rights3;
875
876 private Rights getSalvadorCopyright(String vol) {
877 initRights();
878 if ("2009".equals(vol)){
879 return rights1;
880 }else if ("2012".equals(vol)){
881 return rights1;
882 }else if ("2016".equals(vol)){
883 return rights3;
884 }else{
885 throw new RuntimeException("Volume not recognized: " + vol);
886 }
887 }
888
889 private void initRights(){
890 if (rights1 == null){
891 String text = "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2009.";
892 rights1 = Rights.NewInstance(text, Language.SPANISH_CASTILIAN(), RightsType.COPYRIGHT());
893 text = "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2012.";
894 rights2 = Rights.NewInstance(text, Language.SPANISH_CASTILIAN(), RightsType.COPYRIGHT());
895 text = "(c) Jardín Botánico y Museo Botánico Berlin-Dahlem & Asociación Jardín Botánico La Laguna. Berlin, Antiguo Cuscatlán 2016.";
896 rights3 = Rights.NewInstance(text, Language.SPANISH_CASTILIAN(), RightsType.COPYRIGHT());
897 getCommonService().save(rights1);
898 getCommonService().save(rights2);
899 getCommonService().save(rights3);
900 }
901 }
902
903 private Integer salvadorArtistId;
904 private AgentBase<?> getSalvadorArtist() {
905 if (salvadorArtistId == null){
906 Person person = Person.NewInstance();
907 person.setGivenName("José Gerver");
908 person.setFamilyName("Molina");
909 salvadorArtistId = getAgentService().save(person).getId();
910 return person;
911 }else{
912 return getAgentService().find(salvadorArtistId);
913 }
914 }
915
916 private Integer salvadorRef1Id;
917 private Integer salvadorRef2Id;
918 private Integer salvadorRef3Id;
919
920 private Reference getSalvadorReference(String vol){
921 if (salvadorRef1Id == null){
922 makeSalvadorReferences();
923 }
924 if ("2009".equals(vol)){
925 return getReferenceService().find(salvadorRef1Id);
926 }else if ("2012".equals(vol)){
927 return getReferenceService().find(salvadorRef2Id);
928 }else if ("2016".equals(vol)){
929 return getReferenceService().find(salvadorRef3Id);
930 }else{
931 throw new RuntimeException("Volume not recognized: " + vol);
932 }
933
934 }
935
936 private void makeSalvadorReferences() {
937 Person walter = Person.NewTitledInstance("Berendsohn, W. G.");
938 walter.setGivenName("Walter G.");
939 walter.setFamilyName("Berendsohn");
940 Person katja = Person.NewTitledInstance("Gruber, Anne Kathrina");
941 katja.setGivenName("Anne Katharina");
942 katja.setFamilyName("Gruber");
943 Person monte = Person.NewTitledInstance("Monterrosa Salomón, J.");
944 Person olmedo = Person.NewTitledInstance("Olmedo Galán, P.");
945 Person rodriguez = Person.NewTitledInstance("Rodríguez Delcid, D");
946
947 Team team1 = Team.NewInstance();
948 team1.addTeamMember(walter);
949 team1.addTeamMember(katja);
950 team1.addTeamMember(monte);
951
952 Team team2 = Team.NewInstance();
953 team2.addTeamMember(walter);
954 team2.addTeamMember(katja);
955 team2.addTeamMember(rodriguez);
956 team2.addTeamMember(olmedo);
957
958 Reference vol1 = ReferenceFactory.newBook();
959 Reference vol2 = ReferenceFactory.newBook();
960 Reference vol3 = ReferenceFactory.newBook();
961
962 vol1.setAuthorship(team1);
963 vol2.setAuthorship(team1);
964 vol3.setAuthorship(team2);
965
966 vol1.setDatePublished(TimePeriodParser.parseStringVerbatim("2009"));
967 vol2.setDatePublished(TimePeriodParser.parseStringVerbatim("2012"));
968 vol3.setDatePublished(TimePeriodParser.parseStringVerbatim("2016"));
969
970 Reference englera = ReferenceFactory.newPrintSeries("Englera");
971 vol1.setInSeries(englera);
972 vol2.setInSeries(englera);
973 vol3.setInSeries(englera);
974
975 vol1.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 1: Angiospermae - Familias A a L");
976 vol2.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 2: Angiospermae - Familias M a P y Pteridophyta");
977 vol3.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 3: Angiospermae - Familias R a Z y Gymnospermae");
978
979 vol1.setVolume("29(1)");
980 vol2.setVolume("29(2)");
981 vol3.setVolume("29(3)");
982
983 vol1.setPages("1-438");
984 vol2.setVolume("1-300");
985 vol3.setVolume("1-356");
986
987 String placePublished = "Berlin: Botanic Garden and Botanical Museum Berlin; Antiguo Cuscatlán: Asociación Jardín Botánico La Laguna, El Salvador";
988 vol1.setPlacePublished(placePublished);
989 vol2.setPlacePublished(placePublished);
990 vol3.setPlacePublished(placePublished);
991
992 salvadorRef1Id = getReferenceService().save(vol1).getId();
993 salvadorRef2Id = getReferenceService().find(getReferenceService().saveOrUpdate(vol2)).getId();
994 salvadorRef3Id = getReferenceService().find(getReferenceService().saveOrUpdate(vol3)).getId();
995 return;
996 }
997
998
999 /**
1000 * @param media
1001 * @param uri
1002 * @param imageInfo
1003 * @param size
1004 */
1005 private void makeMediaRepresentation(Media media, URI uri) {
1006 ImageInfo imageInfo = null;
1007 Integer size = null;
1008 try {
1009 imageInfo = ImageInfo.NewInstance(uri, 30);
1010 } catch (IOException | HttpException e) {
1011 logger.error("Error when reading image meta: " + e + ", "+ uri.toString());
1012 }
1013 String mimeType = imageInfo == null ? null : imageInfo.getMimeType();
1014 String suffix = imageInfo == null ? null : imageInfo.getSuffix();
1015 MediaRepresentation mediaRepresentation = MediaRepresentation.NewInstance(mimeType, suffix);
1016 media.addRepresentation(mediaRepresentation);
1017 ImageFile image = ImageFile.NewInstance(uri, size, imageInfo);
1018 mediaRepresentation.addRepresentationPart(image);
1019 }
1020
1021 private TaxonBase<?> getTaxon(Map<String, TaxonBase> taxonMap, Integer taxonIdObj, Number taxonId){
1022 if (taxonIdObj != null){
1023 return taxonMap.get(String.valueOf(taxonId));
1024 }else{
1025 return null;
1026 }
1027
1028 }
1029
1030 private Feature getFeature(Map<Integer, Feature> featureMap, Integer categoryFkInt){
1031 if (categoryFkInt != null){
1032 return featureMap.get(categoryFkInt);
1033 }else{
1034 return null;
1035 }
1036
1037 }
1038
1039 @Override
1040 protected boolean doCheck(BerlinModelImportState state){
1041 IOValidator<BerlinModelImportState> validator = new BerlinModelFactsImportValidator();
1042 return validator.validate(state);
1043 }
1044
1045 @Override
1046 protected boolean isIgnore(BerlinModelImportState state){
1047 return ! state.getConfig().isDoFacts();
1048 }
1049
1050
1051
1052 }