1
|
/*
|
2
|
* Copyright EDIT
|
3
|
* European Distributed Institute of Taxonomy
|
4
|
* http://www.e-taxonomy.eu
|
5
|
*
|
6
|
* The contents of this file are subject to the Mozilla Public License Version 1.1
|
7
|
* See LICENSE.TXT at the top of this package for the full license terms.
|
8
|
*/
|
9
|
package eu.etaxonomy.cdm.remote.controller.checklist;
|
10
|
|
11
|
import java.io.File;
|
12
|
import java.io.IOException;
|
13
|
import java.io.InputStream;
|
14
|
import java.math.BigInteger;
|
15
|
import java.security.MessageDigest;
|
16
|
import java.security.NoSuchAlgorithmException;
|
17
|
import java.util.Arrays;
|
18
|
import java.util.HashMap;
|
19
|
import java.util.List;
|
20
|
import java.util.Map;
|
21
|
import java.util.UUID;
|
22
|
|
23
|
import javax.servlet.http.HttpServletRequest;
|
24
|
import javax.servlet.http.HttpServletResponse;
|
25
|
|
26
|
import org.apache.log4j.Logger;
|
27
|
import org.springframework.beans.factory.annotation.Autowired;
|
28
|
import org.springframework.context.ApplicationContext;
|
29
|
import org.springframework.context.ResourceLoaderAware;
|
30
|
import org.springframework.core.io.Resource;
|
31
|
import org.springframework.core.io.ResourceLoader;
|
32
|
import org.springframework.stereotype.Controller;
|
33
|
import org.springframework.web.bind.WebDataBinder;
|
34
|
import org.springframework.web.bind.annotation.InitBinder;
|
35
|
import org.springframework.web.bind.annotation.RequestMapping;
|
36
|
import org.springframework.web.bind.annotation.RequestMethod;
|
37
|
import org.springframework.web.bind.annotation.RequestParam;
|
38
|
import org.springframework.web.servlet.ModelAndView;
|
39
|
|
40
|
import eu.etaxonomy.cdm.api.service.IClassificationService;
|
41
|
import eu.etaxonomy.cdm.api.service.IService;
|
42
|
import eu.etaxonomy.cdm.api.service.ITaxonNodeService;
|
43
|
import eu.etaxonomy.cdm.common.CdmUtils;
|
44
|
import eu.etaxonomy.cdm.common.DocUtils;
|
45
|
import eu.etaxonomy.cdm.common.monitor.IRestServiceProgressMonitor;
|
46
|
import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
|
47
|
import eu.etaxonomy.cdm.io.common.CdmApplicationAwareDefaultExport;
|
48
|
import eu.etaxonomy.cdm.io.dwca.out.DwcaEmlRecord;
|
49
|
import eu.etaxonomy.cdm.io.dwca.out.DwcaTaxExportConfigurator;
|
50
|
import eu.etaxonomy.cdm.model.common.CdmBase;
|
51
|
import eu.etaxonomy.cdm.model.taxon.Classification;
|
52
|
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
|
53
|
import eu.etaxonomy.cdm.remote.controller.AbstractController;
|
54
|
import eu.etaxonomy.cdm.remote.controller.ProgressMonitorController;
|
55
|
import eu.etaxonomy.cdm.remote.controller.util.ProgressMonitorUtil;
|
56
|
import eu.etaxonomy.cdm.remote.editor.UUIDListPropertyEditor;
|
57
|
import eu.etaxonomy.cdm.remote.editor.UuidList;
|
58
|
import eu.etaxonomy.cdm.remote.view.FileDownloadView;
|
59
|
import eu.etaxonomy.cdm.remote.view.HtmlView;
|
60
|
|
61
|
/**
|
62
|
* @author a.mueller
|
63
|
* @since 28.06.2017
|
64
|
* <p>
|
65
|
* This controller exports taxonomies via Darwin Core Archive
|
66
|
* (https://en.wikipedia.org/wiki/Darwin_Core_Archive).
|
67
|
*/
|
68
|
@Controller
|
69
|
@RequestMapping(value = { "/dwca" })
|
70
|
public class DwcaExportController
|
71
|
extends AbstractController<CdmBase,IService<CdmBase>>
|
72
|
implements ResourceLoaderAware{
|
73
|
|
74
|
|
75
|
private static final String DWCATAX = "dwcatax_";
|
76
|
|
77
|
private static final String DWCA_TAX_EXPORT_DOC_RESSOURCE = "classpath:eu/etaxonomy/cdm/doc/remote/apt/dwca-tax-export-default.apt";
|
78
|
|
79
|
private static final List<String> TAXON_NODE_INIT_STRATEGY = Arrays.asList(new String []{
|
80
|
"taxon.name",
|
81
|
"classification"
|
82
|
});
|
83
|
|
84
|
@Autowired
|
85
|
private ApplicationContext appContext;
|
86
|
|
87
|
@Autowired
|
88
|
private IClassificationService classificationService;
|
89
|
|
90
|
@Autowired
|
91
|
private ITaxonNodeService taxonNodeService;
|
92
|
|
93
|
@Autowired
|
94
|
public ProgressMonitorController progressMonitorController;
|
95
|
|
96
|
private ResourceLoader resourceLoader;
|
97
|
|
98
|
|
99
|
/**
|
100
|
* There should only be one processes operating on the export
|
101
|
* therefore the according progress monitor uuid is stored in this static
|
102
|
* field.
|
103
|
*/
|
104
|
private static UUID indexMonitorUuid = null;
|
105
|
|
106
|
private final static long MINUTE_IN_MILLIS = 60000;
|
107
|
private final static long HOUR_IN_MILLIS = MINUTE_IN_MILLIS * 60;
|
108
|
|
109
|
private final static long DAY_IN_MILLIS = HOUR_IN_MILLIS * 24;
|
110
|
|
111
|
|
112
|
|
113
|
private static final Logger logger = Logger.getLogger(DwcaExportController.class);
|
114
|
|
115
|
/**
|
116
|
* Helper method, which allows to convert strings directly into uuids.
|
117
|
*
|
118
|
* @param binder Special DataBinder for data binding from web request parameters to JavaBean objects.
|
119
|
*/
|
120
|
@InitBinder
|
121
|
public void initBinder(WebDataBinder binder) {
|
122
|
binder.registerCustomEditor(UuidList.class, new UUIDListPropertyEditor());
|
123
|
// binder.registerCustomEditor(NamedArea.class, new TermBaseListPropertyEditor<>(termService));
|
124
|
}
|
125
|
|
126
|
|
127
|
|
128
|
|
129
|
/**
|
130
|
* Documentation webservice for this controller.
|
131
|
*
|
132
|
* @param response unused
|
133
|
* @param request unused
|
134
|
* @return
|
135
|
* @throws IOException
|
136
|
*/
|
137
|
@RequestMapping(value = {""}, method = { RequestMethod.GET})
|
138
|
public ModelAndView exportGetExplanation(HttpServletResponse response,
|
139
|
HttpServletRequest request) throws IOException{
|
140
|
ModelAndView mv = new ModelAndView();
|
141
|
// Read apt documentation file.
|
142
|
Resource resource = resourceLoader.getResource(DWCA_TAX_EXPORT_DOC_RESSOURCE);
|
143
|
// using input stream as this works for both files in the classes directory
|
144
|
// as well as files inside jars
|
145
|
InputStream aptInputStream = resource.getInputStream();
|
146
|
// Build Html View
|
147
|
Map<String, String> modelMap = new HashMap<>();
|
148
|
// Convert Apt to Html
|
149
|
modelMap.put("html", DocUtils.convertAptToHtml(aptInputStream));
|
150
|
mv.addAllObjects(modelMap);
|
151
|
|
152
|
HtmlView hv = new HtmlView();
|
153
|
mv.setView(hv);
|
154
|
return mv;
|
155
|
}
|
156
|
|
157
|
/**
|
158
|
* This service endpoint is for generating the documentation site.
|
159
|
* If any request of the other endpoint below is incomplete or false
|
160
|
* then this method will be triggered.
|
161
|
*
|
162
|
* @param response
|
163
|
* @param request
|
164
|
* @return
|
165
|
* @throws IOException
|
166
|
*/
|
167
|
public ModelAndView exportGetExplanation(HttpServletResponse response,
|
168
|
HttpServletRequest request, Resource res) throws IOException{
|
169
|
ModelAndView mv = new ModelAndView();
|
170
|
// Read apt documentation file.
|
171
|
Resource resource = (res!= null) ? res : resourceLoader.getResource(DWCA_TAX_EXPORT_DOC_RESSOURCE);
|
172
|
// using input stream as this works for both files in the classes directory
|
173
|
// as well as files inside jars
|
174
|
InputStream aptInputStream = resource.getInputStream();
|
175
|
// Build Html View
|
176
|
Map<String, String> modelMap = new HashMap<>();
|
177
|
// Convert Apt to Html
|
178
|
modelMap.put("html", DocUtils.convertAptToHtml(aptInputStream));
|
179
|
mv.addAllObjects(modelMap);
|
180
|
|
181
|
HtmlView hv = new HtmlView();
|
182
|
mv.setView(hv);
|
183
|
return mv;
|
184
|
}
|
185
|
|
186
|
|
187
|
|
188
|
/**
|
189
|
*
|
190
|
* This Service endpoint will offer a csv file. It caches the csv-file in the system temp directory
|
191
|
* and will only generate a new one after 24 hours. Or if explicitly triggerd by noCache parameter.
|
192
|
*
|
193
|
* @param featureUuids List of uuids to download/select {@link Feature feature}features
|
194
|
* @param clearCache will trigger export and avoids cached file
|
195
|
* @param classificationUUID Selected {@link Classification classification} to iterate the {@link Taxon}
|
196
|
* @param response HttpServletResponse which returns the ByteArrayOutputStream
|
197
|
* @throws Exception
|
198
|
*/
|
199
|
@RequestMapping(value = { "dwcaTaxExport" }, method = { RequestMethod.GET })
|
200
|
public synchronized ModelAndView doDwcaTaxExport(
|
201
|
@RequestParam(value = "subtrees", required = false) final UuidList subtreeUuids,
|
202
|
@RequestParam(value = "clearCache", required = false) final boolean clearCache,
|
203
|
@RequestParam(value = "classifications", required = false) final UuidList classificationUuids,
|
204
|
@RequestParam(value = "taxa", required = false) final UuidList taxonUuids,
|
205
|
@RequestParam(value = "taxonnodes", required = false) final UuidList taxonNodeUuids,
|
206
|
@RequestParam(value = "doMisapplieds", defaultValue="true") Boolean doMisapplieds,
|
207
|
@RequestParam(value = "doSynonyms", defaultValue="true") Boolean doSynonyms,
|
208
|
@RequestParam(value = "doImages", defaultValue="true") Boolean doImages,
|
209
|
@RequestParam(value = "doDescriptions", defaultValue="true") Boolean doDescriptions,
|
210
|
@RequestParam(value = "doDistributions", defaultValue="true") Boolean doDistributions,
|
211
|
@RequestParam(value = "doVernaculars", defaultValue="true") Boolean doVernaculars,
|
212
|
@RequestParam(value = "doTypesAndSpecimen", defaultValue="true") Boolean doTypesAndSpecimen,
|
213
|
@RequestParam(value = "doResourceRelations", defaultValue="true") Boolean doResourceRelations,
|
214
|
@RequestParam(value = "doReferences", defaultValue="true") Boolean doReferences,
|
215
|
@RequestParam(value = "withHigherClassification", defaultValue="false") Boolean withHigherClassification,
|
216
|
@RequestParam(value = "includeHeader", defaultValue="false") Boolean includeHeader,
|
217
|
// @RequestParam(value = "includeUnpublished", defaultValue="false") Boolean includeUnpublished, //for now we do not allow unpublished data to be exported via webservice as long as read authentication is not implemented
|
218
|
|
219
|
@RequestParam(value = "area", required = false) final UuidList areaUuids,
|
220
|
@RequestParam(value = "minRank", required = false) final UUID minRank,
|
221
|
@RequestParam(value = "minRank", required = false) final UUID maxRank,
|
222
|
|
223
|
@RequestParam(value = "downloadTokenValueId", required = false) final String downloadTokenValueId,
|
224
|
@RequestParam(value = "priority", required = false) Integer priority,
|
225
|
final HttpServletResponse response,
|
226
|
final HttpServletRequest request) throws Exception {
|
227
|
/**
|
228
|
* ========================================
|
229
|
* progress monitor & new thread for export
|
230
|
* ========================================
|
231
|
*/
|
232
|
try{
|
233
|
ModelAndView mv = new ModelAndView();
|
234
|
|
235
|
// replacement for commented RequestParam
|
236
|
Boolean includeUnpublished = false;
|
237
|
|
238
|
final String origin = request.getRequestURL().append('?')
|
239
|
.append(CdmUtils.Nz(request.getQueryString())).toString()
|
240
|
.replace("&clearCache=true", "").replace("?clearCache=true", "?");
|
241
|
|
242
|
String fileName = makeFileName(response, origin, DWCATAX);
|
243
|
final File cacheFile = new File(new File(System.getProperty("java.io.tmpdir")), fileName);
|
244
|
|
245
|
Long result = null;
|
246
|
if(cacheFile.exists()){
|
247
|
result = System.currentTimeMillis() - cacheFile.lastModified();
|
248
|
}
|
249
|
//if file exists return file instantly
|
250
|
//timestamp older than one day?
|
251
|
if(clearCache == false && result != null){ //&& result < 7*(DAY_IN_MILLIS)
|
252
|
logger.info("result of calculation: " + result);
|
253
|
Map<String, File> modelMap = new HashMap<>();
|
254
|
modelMap.put("file", cacheFile);
|
255
|
mv.addAllObjects(modelMap);
|
256
|
FileDownloadView fdv = new FileDownloadView(fileName, "zip");
|
257
|
mv.setView(fdv);
|
258
|
return mv;
|
259
|
}else{//trigger progress monitor and performExport()
|
260
|
String processLabel = "Exporting ...";
|
261
|
final String frontbaseUrl = null;
|
262
|
ProgressMonitorUtil progressUtil = new ProgressMonitorUtil(progressMonitorController);
|
263
|
if (!progressMonitorController.isMonitorRunning(indexMonitorUuid)) {
|
264
|
indexMonitorUuid = progressUtil.registerNewMonitor();
|
265
|
Thread subThread = new Thread() {
|
266
|
@Override
|
267
|
public void run() {
|
268
|
try {
|
269
|
boolean created = cacheFile.createNewFile();
|
270
|
// boolean created = cacheFile.mkdir();
|
271
|
if (!created){logger.info("Could not create file");}
|
272
|
} catch (Exception e) {
|
273
|
logger.info("Could not create file " + e);
|
274
|
}
|
275
|
IRestServiceProgressMonitor monitor = progressMonitorController.getMonitor(
|
276
|
indexMonitorUuid);
|
277
|
|
278
|
TaxonNodeFilter taxonNodeFilter = TaxonNodeFilter.NewInstance(
|
279
|
classificationUuids, subtreeUuids, taxonNodeUuids, taxonUuids,
|
280
|
areaUuids, minRank, maxRank);
|
281
|
taxonNodeFilter.setIncludeUnpublished(includeUnpublished);
|
282
|
DwcaTaxExportConfigurator config = setDwcaTaxExportConfigurator(
|
283
|
cacheFile, monitor, taxonNodeFilter, doSynonyms, doMisapplieds,
|
284
|
doVernaculars, doDistributions, doDescriptions, doImages,
|
285
|
doTypesAndSpecimen, doResourceRelations, doReferences,
|
286
|
withHigherClassification, includeHeader);
|
287
|
performExport(cacheFile, monitor, config,
|
288
|
downloadTokenValueId, origin, response);
|
289
|
}
|
290
|
};
|
291
|
if (priority == null) {
|
292
|
priority = AbstractController.DEFAULT_BATCH_THREAD_PRIORITY;
|
293
|
}
|
294
|
subThread.setPriority(priority);
|
295
|
subThread.start();
|
296
|
}
|
297
|
mv = progressUtil.respondWithMonitorOrDownload(frontbaseUrl, origin, processLabel, indexMonitorUuid, false, request, response);
|
298
|
}
|
299
|
return mv;
|
300
|
}catch(Exception e){
|
301
|
//TODO: Write an specific documentation for this service endpoint
|
302
|
Resource resource = resourceLoader.getResource(DWCA_TAX_EXPORT_DOC_RESSOURCE);
|
303
|
return exportGetExplanation(response, request, resource);
|
304
|
}
|
305
|
}
|
306
|
|
307
|
|
308
|
|
309
|
//=========== Helper Methods ===============//
|
310
|
|
311
|
/**
|
312
|
* Creates an (MD5) Hash of the URI and uses it for the local file name, to be unique
|
313
|
* @param response
|
314
|
* @param origin
|
315
|
* @param prefix
|
316
|
* @return
|
317
|
* @throws IOException
|
318
|
*/
|
319
|
private String makeFileName(HttpServletResponse response, String origin, String prefix) throws IOException {
|
320
|
try {
|
321
|
MessageDigest md = MessageDigest.getInstance("MD5");
|
322
|
md.update(origin.getBytes(),0,origin.length());
|
323
|
String result = new BigInteger(1,md.digest()).toString(16);
|
324
|
return prefix + result;
|
325
|
} catch (NoSuchAlgorithmException e) {
|
326
|
String message = "Can't create temporary filename hash for " + origin;
|
327
|
response.sendError(404, message);
|
328
|
throw new RuntimeException(message, e);
|
329
|
}
|
330
|
}
|
331
|
|
332
|
|
333
|
|
334
|
/**
|
335
|
*
|
336
|
* This private methods finally triggers the export back in the io-package and will create a cache file
|
337
|
* in system temp directory.
|
338
|
*
|
339
|
* @param downloadTokenValueId
|
340
|
* @param response
|
341
|
* @param byteArrayOutputStream
|
342
|
* @param config
|
343
|
* @param defaultExport
|
344
|
*/
|
345
|
private void performExport(File cacheFile, IRestServiceProgressMonitor progressMonitor,
|
346
|
DwcaTaxExportConfigurator config,
|
347
|
String downloadTokenValueId, String origin,
|
348
|
HttpServletResponse response
|
349
|
) {
|
350
|
|
351
|
progressMonitor.subTask("configure export");
|
352
|
@SuppressWarnings("unchecked")
|
353
|
CdmApplicationAwareDefaultExport<DwcaTaxExportConfigurator> defaultExport =
|
354
|
(CdmApplicationAwareDefaultExport<DwcaTaxExportConfigurator>)appContext.getBean("defaultExport");
|
355
|
progressMonitor.beginTask("DwC-A export", 100);
|
356
|
progressMonitor.subTask("invoke export");
|
357
|
defaultExport.invoke(config); //triggers export
|
358
|
progressMonitor.subTask("wrote results to cache");
|
359
|
progressMonitor.done();
|
360
|
progressMonitor.setOrigin(origin);
|
361
|
}
|
362
|
|
363
|
/**
|
364
|
* Cofiguration method to set the configuration details for the defaultExport in the application context.
|
365
|
* @param cacheFile
|
366
|
* @param progressMonitor
|
367
|
* @param doImages
|
368
|
* @param doDescriptions
|
369
|
* @param doDistributions
|
370
|
* @param doVernaculars
|
371
|
* @param doReferences
|
372
|
* @param doResourceRelations
|
373
|
* @param doTypesAndSpecimen
|
374
|
* @param withHigherClassification
|
375
|
* @param includeHeader
|
376
|
*/
|
377
|
private DwcaTaxExportConfigurator setDwcaTaxExportConfigurator(File cacheFile,
|
378
|
IRestServiceProgressMonitor progressMonitor,
|
379
|
TaxonNodeFilter taxonNodeFilter,
|
380
|
boolean doSynonyms, boolean doMisappliedNames,
|
381
|
Boolean doVernaculars, Boolean doDistributions,
|
382
|
Boolean doDescriptions, Boolean doImages,
|
383
|
Boolean doTypesAndSpecimen, Boolean doResourceRelations,
|
384
|
Boolean doReferences, Boolean withHigherClassification,
|
385
|
Boolean includeHeader) {
|
386
|
|
387
|
if(cacheFile == null){
|
388
|
String destination = System.getProperty("java.io.tmpdir");
|
389
|
cacheFile = new File(destination);
|
390
|
}
|
391
|
|
392
|
DwcaEmlRecord emlRecord = null;
|
393
|
DwcaTaxExportConfigurator config = DwcaTaxExportConfigurator.NewInstance(
|
394
|
null, cacheFile, emlRecord);
|
395
|
|
396
|
config.setTaxonNodeFilter(taxonNodeFilter);
|
397
|
config.setDoSynonyms(doSynonyms);
|
398
|
config.setDoMisappliedNames(doMisappliedNames);
|
399
|
|
400
|
config.setDoDescriptions(doDescriptions);
|
401
|
config.setDoVernacularNames(doVernaculars);
|
402
|
config.setDoImages(doImages);
|
403
|
config.setDoDistributions(doDistributions);
|
404
|
config.setDoTypesAndSpecimen(doTypesAndSpecimen);
|
405
|
config.setDoReferences(doReferences);
|
406
|
config.setDoResourceRelations(doResourceRelations);
|
407
|
config.setWithHigherClassification(withHigherClassification);
|
408
|
config.setHasHeaderLines(includeHeader);
|
409
|
|
410
|
config.setProgressMonitor(progressMonitor);
|
411
|
|
412
|
// config.setHasHeaderLines(true);
|
413
|
// config.setFieldsTerminatedBy("\t");
|
414
|
// config.setClassificationUuids(classificationUUIDS);
|
415
|
|
416
|
// if(demoExport == false && conceptExport == false){
|
417
|
// config.createPreSelectedExport(false, true);
|
418
|
// }else{
|
419
|
// config.createPreSelectedExport(demoExport, conceptExport);
|
420
|
// }
|
421
|
|
422
|
return config;
|
423
|
}
|
424
|
|
425
|
|
426
|
/**
|
427
|
* @param response
|
428
|
* @param subtreeUuids
|
429
|
* @throws IOException
|
430
|
*/
|
431
|
private String makeFileNameOld(HttpServletResponse response, UuidList subtreeUuids) throws IOException {
|
432
|
String fileName;
|
433
|
if (subtreeUuids != null && ! subtreeUuids.isEmpty()){
|
434
|
UUID firstUuid = subtreeUuids.get(0);
|
435
|
TaxonNode node = taxonNodeService.load(firstUuid, TAXON_NODE_INIT_STRATEGY);
|
436
|
if (node != null && node.getTaxon() != null){
|
437
|
if (node.getTaxon().getName() != null){
|
438
|
fileName = node.getTaxon().getName().getTitleCache();
|
439
|
}else{
|
440
|
fileName = node.getTaxon().getTitleCache();
|
441
|
}
|
442
|
}else if (node != null){
|
443
|
fileName = node.getClassification().getTitleCache();
|
444
|
}else{
|
445
|
Classification classification = classificationService.find(firstUuid);
|
446
|
if (classification != null){
|
447
|
fileName = classification.getTitleCache();
|
448
|
}else{
|
449
|
//handle via repso
|
450
|
response.sendError(404, "Subtree uuid does not exist: " + firstUuid);
|
451
|
fileName = "Error";
|
452
|
}
|
453
|
}
|
454
|
}else{
|
455
|
List<Classification> classificationList = classificationService.list(null, 1, null
|
456
|
, null, null);
|
457
|
if (!classificationList.isEmpty()){
|
458
|
fileName = classificationList.get(0).getTitleCache();
|
459
|
}else{
|
460
|
//handle via repso
|
461
|
response.sendError(404, "No classification found");
|
462
|
fileName = "Error";
|
463
|
}
|
464
|
}
|
465
|
|
466
|
|
467
|
fileName = fileName + "_" + uuidListToString(subtreeUuids, 40);
|
468
|
|
469
|
return fileName;
|
470
|
|
471
|
}
|
472
|
|
473
|
private String uuidListToString(UuidList uuidList, Integer truncate) {
|
474
|
String result = null;
|
475
|
if (uuidList != null){
|
476
|
for (UUID uuid : uuidList){
|
477
|
result = CdmUtils.concat("_", uuid.toString());
|
478
|
}
|
479
|
}
|
480
|
if (result != null && result.length() > truncate){
|
481
|
result = result.substring(0, truncate);
|
482
|
}
|
483
|
return result;
|
484
|
}
|
485
|
|
486
|
@Override
|
487
|
public void setService(IService service) {}
|
488
|
|
489
|
@Override
|
490
|
public void setResourceLoader(ResourceLoader resourceLoader) {
|
491
|
this.resourceLoader = resourceLoader;
|
492
|
}
|
493
|
|
494
|
}
|