Project

General

Profile

Revision 00dfef96

ID00dfef968d014f640fc5a69feb3163d6ffba80ac
Parent 03eb176d
Child 18a0110f

Added by Andreas Müller over 3 years ago

ref #6755 first preliminary version of DwC-A export web service

View differences:

cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/dwca/out/DwcaExportBase.java
29 29

  
30 30
import org.apache.commons.lang.StringUtils;
31 31
import org.apache.log4j.Logger;
32
import org.springframework.beans.factory.annotation.Autowired;
32 33

  
34
import eu.etaxonomy.cdm.api.service.IClassificationService;
35
import eu.etaxonomy.cdm.api.service.ITaxonNodeService;
33 36
import eu.etaxonomy.cdm.common.CdmUtils;
34 37
import eu.etaxonomy.cdm.io.common.CdmExportBase;
35 38
import eu.etaxonomy.cdm.io.common.ICdmExport;
......
61 64

  
62 65
    protected static final boolean IS_CORE = true;
63 66

  
67
    @Autowired
68
    private IClassificationService classificationService;
69

  
70
    @Autowired
71
    private ITaxonNodeService taxonNodeService;
72

  
64 73

  
65 74
    @Override
66 75
    public int countSteps(DwcaTaxExportState state) {
......
104 113

  
105 114
    private void makeAllNodes(DwcaTaxExportState state, Set<UUID> subtreeSet) {
106 115

  
107
        boolean doSynonyms = false;
108
        boolean recursive = true;
109
        Set<UUID> uuidSet = new HashSet<>();
116
        try {
117
            boolean doSynonyms = false;
118
            boolean recursive = true;
119
            Set<UUID> uuidSet = new HashSet<>();
120

  
121
            for (UUID subtreeUuid : subtreeSet){
122
                UUID tnUuuid = taxonNodeUuid(subtreeUuid);
123
                uuidSet.add(tnUuuid);
124
                List<TaxonNodeDto> records = getTaxonNodeService().pageChildNodesDTOs(tnUuuid,
125
                        recursive, doSynonyms, null, null, null).getRecords();
126
                for (TaxonNodeDto dto : records){
127
                    uuidSet.add(dto.getUuid());
128
                }
129
            }
130
            List<TaxonNode> allNodes =  getTaxonNodeService().find(uuidSet);
110 131

  
111
        for (UUID subtreeUuid : subtreeSet){
112
            uuidSet.add(subtreeUuid);
113
            List<TaxonNodeDto> records = getTaxonNodeService().pageChildNodesDTOs(subtreeUuid,
114
                    recursive, doSynonyms, null, null, null).getRecords();
115
            for (TaxonNodeDto dto : records){
116
                uuidSet.add(dto.getUuid());
132
            List<TaxonNode> result = new ArrayList<>();
133
            for (TaxonNode node : allNodes){
134
                if(node.getParent()== null){  //root (or invalid) node
135
                    continue;
136
                }
137
                node = CdmBase.deproxy(node);
138
                Taxon taxon = CdmBase.deproxy(node.getTaxon());
139
                if (taxon == null){
140
                    String message = "There is a taxon node without taxon. id=" + node.getId();
141
                    state.getResult().addWarning(message);
142
                    continue;
143
                }
144
                result.add(node);
117 145
            }
146
            state.setAllNodes(result);
147
        } catch (Exception e) {
148
            String message = "Unexpected exception when trying to compute all taxon nodes";
149
            state.getResult().addException(e, message);
118 150
        }
119
        List<TaxonNode> allNodes =  getTaxonNodeService().find(uuidSet);
151
    }
120 152

  
121
        List<TaxonNode> result = new ArrayList<>();
122
        for (TaxonNode node : allNodes){
123
            if(node.getParent()== null){  //root (or invalid) node
124
                continue;
125
            }
126
            node = CdmBase.deproxy(node);
127
            Taxon taxon = CdmBase.deproxy(node.getTaxon());
128
            if (taxon == null){
129
                String message = "There is a taxon node without taxon. id=" + node.getId();
130
                state.getResult().addWarning(message);
131
                continue;
153

  
154
    /**
155
     * @param subtreeUuid
156
     * @return
157
     */
158
    private UUID taxonNodeUuid(UUID subtreeUuid) {
159
        TaxonNode node = taxonNodeService.find(subtreeUuid);
160
        if (node == null){
161
            Classification classification = classificationService.find(subtreeUuid);
162
            if (classification != null){
163
                node = classification.getRootNode();
164
            }else{
165
                throw new IllegalArgumentException("Subtree identifier does not exist: " + subtreeUuid);
132 166
            }
133
            result.add(node);
134 167
        }
135
        state.setAllNodes(result);
168
        return node.getUuid();
136 169
    }
137 170

  
138

  
139 171
    /**
140 172
     * Creates the locationId, locality, countryCode triple
141 173
     * @param record
cdmlib-persistence/src/main/java/eu/etaxonomy/cdm/persistence/dto/TaxonNodeDto.java
78 78
        uuid = taxonNode.getUuid();
79 79
        taxonomicChildrenCount = taxonNode.getCountChildren();
80 80
        Taxon taxon = taxonNode.getTaxon();
81
        secUuid = taxon.getSec().getUuid();
81
        secUuid = taxon.getSec() != null ? taxon.getSec().getUuid() : null;
82 82
        taxonUuid = taxon.getUuid();
83
        titleCache = taxon.getName().getTitleCache();
84
        taggedTitle = taxon.getName().getTaggedName();
83
        titleCache = taxon.getName() != null ? taxon.getName().getTitleCache() : taxon.getTitleCache();
84
        taggedTitle = taxon.getName() != null? taxon.getName().getTaggedName() : taxon.getTaggedTitle();
85 85
        unplaced = taxonNode.isUnplaced();
86 86
        excluded = taxonNode.isExcluded();
87
        rankLabel = taxon.getName().getRank().getLabel();
87
        rankLabel = taxon.getNullSafeRank() != null ? taxon.getNullSafeRank().getLabel() : null;
88 88
        status = TaxonStatus.Accepted;
89 89
    }
90 90

  
cdmlib-remote/src/main/java/eu/etaxonomy/cdm/remote/controller/checklist/DwcaExportController.java
1
/*
2
* Copyright  EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.remote.controller.checklist;
10

  
11
import java.io.File;
12
import java.io.IOException;
13
import java.io.InputStream;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.List;
17
import java.util.Map;
18
import java.util.Set;
19
import java.util.UUID;
20

  
21
import javax.servlet.http.HttpServletRequest;
22
import javax.servlet.http.HttpServletResponse;
23

  
24
import org.apache.log4j.Logger;
25
import org.springframework.beans.factory.annotation.Autowired;
26
import org.springframework.context.ApplicationContext;
27
import org.springframework.context.ResourceLoaderAware;
28
import org.springframework.core.io.Resource;
29
import org.springframework.core.io.ResourceLoader;
30
import org.springframework.stereotype.Controller;
31
import org.springframework.web.bind.WebDataBinder;
32
import org.springframework.web.bind.annotation.InitBinder;
33
import org.springframework.web.bind.annotation.RequestMapping;
34
import org.springframework.web.bind.annotation.RequestMethod;
35
import org.springframework.web.bind.annotation.RequestParam;
36
import org.springframework.web.servlet.ModelAndView;
37

  
38
import eu.etaxonomy.cdm.api.service.IClassificationService;
39
import eu.etaxonomy.cdm.api.service.IService;
40
import eu.etaxonomy.cdm.api.service.ITaxonNodeService;
41
import eu.etaxonomy.cdm.common.CdmUtils;
42
import eu.etaxonomy.cdm.common.DocUtils;
43
import eu.etaxonomy.cdm.common.monitor.IRestServiceProgressMonitor;
44
import eu.etaxonomy.cdm.io.common.CdmApplicationAwareDefaultExport;
45
import eu.etaxonomy.cdm.io.dwca.out.DwcaEmlRecord;
46
import eu.etaxonomy.cdm.io.dwca.out.DwcaTaxExportConfigurator;
47
import eu.etaxonomy.cdm.model.description.Feature;
48
import eu.etaxonomy.cdm.model.taxon.Classification;
49
import eu.etaxonomy.cdm.model.taxon.Taxon;
50
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
51
import eu.etaxonomy.cdm.remote.controller.AbstractController;
52
import eu.etaxonomy.cdm.remote.controller.ProgressMonitorController;
53
import eu.etaxonomy.cdm.remote.controller.util.ProgressMonitorUtil;
54
import eu.etaxonomy.cdm.remote.editor.UUIDListPropertyEditor;
55
import eu.etaxonomy.cdm.remote.editor.UuidList;
56
import eu.etaxonomy.cdm.remote.view.FileDownloadView;
57
import eu.etaxonomy.cdm.remote.view.HtmlView;
58

  
59
/**
60
 * @author a.mueller
61
 * @created 28.06.2017
62
 * <p>
63
 *  This controller exports taxonomies via Darwin Core Archive
64
 *  (https://en.wikipedia.org/wiki/Darwin_Core_Archive).
65
 */
66
@Controller
67
@RequestMapping(value = { "/dwca" })
68
public class DwcaExportController extends AbstractController implements ResourceLoaderAware{
69

  
70

  
71
    private static final String DWCA_TAX_EXPORT_DOC_RESSOURCE = "classpath:eu/etaxonomy/cdm/doc/remote/apt/dwca-tax-export-default.apt";
72

  
73
    @Autowired
74
    private ApplicationContext appContext;
75

  
76
    @Autowired
77
    private IClassificationService classificationService;
78

  
79
    @Autowired
80
    private ITaxonNodeService taxonNodeService;
81

  
82
    @Autowired
83
    public ProgressMonitorController progressMonitorController;
84

  
85
    private ResourceLoader resourceLoader;
86

  
87

  
88
    /**
89
     * There should only be one processes operating on the export
90
     * therefore the according progress monitor uuid is stored in this static
91
     * field.
92
     */
93
    private static UUID indexMonitorUuid = null;
94

  
95
    private final static long DAY_IN_MILLIS = 86400000;
96

  
97

  
98

  
99
    private static final Logger logger = Logger.getLogger(DwcaExportController.class);
100

  
101
    /**
102
     * Helper method, which allows to convert strings directly into uuids.
103
     *
104
     * @param binder Special DataBinder for data binding from web request parameters to JavaBean objects.
105
     */
106
    @InitBinder
107
    public void initBinder(WebDataBinder binder) {
108
        binder.registerCustomEditor(UuidList.class, new UUIDListPropertyEditor());
109
//        binder.registerCustomEditor(NamedArea.class, new TermBaseListPropertyEditor<>(termService));
110
//        binder.registerCustomEditor(UUID.class, new UUIDEditor());
111
    }
112

  
113

  
114

  
115

  
116
    /**
117
     * Documentation webservice for this controller.
118
     *
119
     * @param response unused
120
     * @param request unused
121
     * @return
122
     * @throws IOException
123
     */
124
    @RequestMapping(value = {""}, method = { RequestMethod.GET})
125
    public ModelAndView exportGetExplanation(HttpServletResponse response,
126
            HttpServletRequest request) throws IOException{
127
        ModelAndView mv = new ModelAndView();
128
        // Read apt documentation file.
129
        Resource resource = resourceLoader.getResource(DWCA_TAX_EXPORT_DOC_RESSOURCE);
130
        // using input stream as this works for both files in the classes directory
131
        // as well as files inside jars
132
        InputStream aptInputStream = resource.getInputStream();
133
        // Build Html View
134
        Map<String, String> modelMap = new HashMap<>();
135
        // Convert Apt to Html
136
        modelMap.put("html", DocUtils.convertAptToHtml(aptInputStream));
137
        mv.addAllObjects(modelMap);
138

  
139
        HtmlView hv = new HtmlView();
140
        mv.setView(hv);
141
        return mv;
142
    }
143

  
144
    /**
145
     * This service endpoint is for generating the documentation site.
146
     * If any request of the other endpoint below is incomplete or false
147
     * then this method will be triggered.
148
     *
149
     * @param response
150
     * @param request
151
     * @return
152
     * @throws IOException
153
     */
154
    public ModelAndView exportGetExplanation(HttpServletResponse response,
155
            HttpServletRequest request, Resource res) throws IOException{
156
        ModelAndView mv = new ModelAndView();
157
        // Read apt documentation file.
158
        Resource resource = (res!= null) ? res : resourceLoader.getResource(DWCA_TAX_EXPORT_DOC_RESSOURCE);
159
        // using input stream as this works for both files in the classes directory
160
        // as well as files inside jars
161
        InputStream aptInputStream = resource.getInputStream();
162
        // Build Html View
163
        Map<String, String> modelMap = new HashMap<>();
164
        // Convert Apt to Html
165
        modelMap.put("html", DocUtils.convertAptToHtml(aptInputStream));
166
        mv.addAllObjects(modelMap);
167

  
168
        HtmlView hv = new HtmlView();
169
        mv.setView(hv);
170
        return mv;
171
    }
172

  
173

  
174

  
175
    /**
176
     *
177
     * This Service endpoint will offer a csv file. It caches the csv-file in the system temp directory
178
     * and will only generate a new one after 24 hours. Or if explicitly triggerd by noCache parameter.
179
     *
180
     * @param featureUuids List of uuids to download/select {@link Feature feature}features
181
     * @param clearCache will trigger export and avoids cached file
182
     * @param classificationUUID Selected {@link Classification classification} to iterate the {@link Taxon}
183
     * @param response HttpServletResponse which returns the ByteArrayOutputStream
184
     * @throws Exception
185
     */
186
    @RequestMapping(value = { "dwcaTaxExport" }, method = { RequestMethod.GET })
187
    public synchronized ModelAndView doDwcaTaxExport(
188
            @RequestParam(value = "subtrees", required = false) final UuidList subtreeUuids,
189
            @RequestParam(value = "clearCache", required = false) final boolean clearCache,
190
//            @RequestParam(value = "demoExport", required = false) final boolean demoExport,
191
//            @RequestParam(value = "conceptExport", required = false) final boolean conceptExport,
192
//            @RequestParam(value = "classification", required = false) final String classificationUUID,
193
//            @RequestParam(value = "area", required = false) final UuidList areas,
194
            @RequestParam(value = "downloadTokenValueId", required = false) final String downloadTokenValueId,
195
            @RequestParam(value = "priority", required = false) Integer priority,
196
            final HttpServletResponse response,
197
            final HttpServletRequest request) throws Exception {
198
        /**
199
         * ========================================
200
         * progress monitor & new thread for export
201
         * ========================================
202
         */
203
        try{
204
            ModelAndView mv = new ModelAndView();
205

  
206
            String fileName = makeFileName(response, subtreeUuids);
207

  
208
            final File cacheFile = new File(new File(System.getProperty("java.io.tmpdir")), fileName);
209
            final String origin = request.getRequestURL().append('?')
210
                    .append(request.getQueryString()).toString();
211

  
212
            Long result = null;
213
            if(cacheFile.exists()){
214
                result = System.currentTimeMillis() - cacheFile.lastModified();
215
            }
216
            //if file exists return file instantly
217
            //timestamp older than one day?
218
            if(clearCache == false && result != null){ //&& result < 7*(DAY_IN_MILLIS)
219
                logger.info("result of calculation: " + result);
220
                Map<String, File> modelMap = new HashMap<>();
221
                modelMap.put("file", cacheFile);
222
                mv.addAllObjects(modelMap);
223
                //application/zip
224
                FileDownloadView fdv = new FileDownloadView("application/octet-stream", fileName, "zip", "UTF-8");
225
                mv.setView(fdv);
226
                return mv;
227
            }else{//trigger progress monitor and performExport()
228
                String processLabel = "Exporting ...";
229
                final String frontbaseUrl = null;
230
                ProgressMonitorUtil progressUtil = new ProgressMonitorUtil(progressMonitorController);
231
                if (!progressMonitorController.isMonitorRunning(indexMonitorUuid)) {
232
                    indexMonitorUuid = progressUtil.registerNewMonitor();
233
                    Thread subThread = new Thread() {
234
                        @Override
235
                        public void run() {
236
                            try {
237
                                cacheFile.createNewFile();
238
                            } catch (IOException e) {
239
                                logger.info("Could not create file "+ e);
240
                            }
241
                            performExport(cacheFile, progressMonitorController.getMonitor(indexMonitorUuid),
242
                                    subtreeUuids, downloadTokenValueId, origin, response);
243
                        }
244
                    };
245
                    if (priority == null) {
246
                        priority = AbstractController.DEFAULT_BATCH_THREAD_PRIORITY;
247
                    }
248
                    subThread.setPriority(priority);
249
                    subThread.start();
250
                }
251
                mv = progressUtil.respondWithMonitorOrDownload(frontbaseUrl, origin, processLabel, indexMonitorUuid, false, request, response);
252
            }
253
            return mv;
254
        }catch(Exception e){
255
            //TODO: Write an specific documentation for this service endpoint
256
           Resource resource = resourceLoader.getResource(DWCA_TAX_EXPORT_DOC_RESSOURCE);
257
           return exportGetExplanation(response, request, resource);
258
        }
259
    }
260

  
261

  
262

  
263
    //=========== Helper Methods ===============//
264

  
265
    /**
266
     *
267
     * This private methods finally triggers the export back in the io-package and will create a cache file
268
     * in system temp directory.
269
     *
270
     * @param downloadTokenValueId
271
     * @param response
272
     * @param byteArrayOutputStream
273
     * @param config
274
     * @param defaultExport
275
     */
276
    private void performExport(File cacheFile, IRestServiceProgressMonitor progressMonitor,
277
            UuidList featureUuids, String downloadTokenValueId, String origin,
278
            HttpServletResponse response
279
            ) {
280

  
281
        progressMonitor.subTask("configure export");
282
        DwcaTaxExportConfigurator config = setDwcaTaxExportConfigurator(cacheFile, progressMonitor, featureUuids);
283
        @SuppressWarnings("unchecked")
284
        CdmApplicationAwareDefaultExport<DwcaTaxExportConfigurator> defaultExport =
285
                (CdmApplicationAwareDefaultExport<DwcaTaxExportConfigurator>)appContext.getBean("defaultExport");
286
        progressMonitor.subTask("invoke export");
287
        defaultExport.invoke(config);  //triggers export
288
        progressMonitor.subTask("wrote results to cache");
289
        progressMonitor.done();
290
        progressMonitor.setOrigin(origin);
291
    }
292

  
293
    /**
294
     * Cofiguration method to set the configuration details for the defaultExport in the application context.
295
     * @param cacheFile
296
     *
297
     * @param classificationUUID pass-through the selected {@link Classification classification}
298
     * @param featureUuids pass-through the selected {@link Feature feature} of a {@link Taxon}, in order to fetch it.
299
     * @param areas
300
     * @param byteArrayOutputStream pass-through the stream to write out the data later.
301
     * @param progressMonitor
302
     * @param conceptExport
303
     * @param demoExport
304
     * @return the CsvTaxExportConfiguratorRedlist config
305
     */
306
    private DwcaTaxExportConfigurator setDwcaTaxExportConfigurator(File cacheFile, IRestServiceProgressMonitor progressMonitor,
307
            UuidList subtreeUuids) {
308

  
309
        if(cacheFile == null){
310
            String destination = System.getProperty("java.io.tmpdir");
311
            cacheFile = new File(destination);
312
        }
313

  
314
        DwcaEmlRecord emlRecord = null;
315
        DwcaTaxExportConfigurator config = DwcaTaxExportConfigurator.NewInstance(
316
                null, cacheFile, emlRecord);
317

  
318
        Set<UUID> subtreeSet = new HashSet<>(subtreeUuids);
319
        config.setProgressMonitor(progressMonitor);
320
        config.setSubtreeUuids(subtreeSet);
321

  
322
//        config.setHasHeaderLines(true);
323
//        config.setFieldsTerminatedBy("\t");
324
//        config.setClassificationUuids(classificationUUIDS);
325

  
326
//        if(demoExport == false && conceptExport == false){
327
//        	config.createPreSelectedExport(false, true);
328
//        }else{
329
//        	config.createPreSelectedExport(demoExport, conceptExport);
330
//        }
331

  
332
        return config;
333
    }
334

  
335

  
336
    /**
337
     * @param response
338
     * @param subtreeUuids
339
     * @throws IOException
340
     */
341
    private String makeFileName(HttpServletResponse response, UuidList subtreeUuids) throws IOException {
342
        String fileName;
343
        if (subtreeUuids != null && ! subtreeUuids.isEmpty()){
344
            UUID firstUuid = subtreeUuids.get(0);
345
            TaxonNode node = taxonNodeService.find(firstUuid);
346
            if (node != null && node.getTaxon() != null){
347
                if (node.getTaxon().getName() != null){
348
                    fileName = node.getTaxon().getName().getTitleCache();
349
                }else{
350
                    fileName = node.getTaxon().getTitleCache();
351
                }
352
            }else if (node != null){
353
                fileName = node.getClassification().getTitleCache();
354
            }else{
355
                Classification classification = classificationService.find(firstUuid);
356
                if (classification != null){
357
                    fileName = classification.getTitleCache();
358
                }else{
359
                    //handle via repso
360
                    response.sendError(404, "Subtree uuid does not exist: " + firstUuid);
361
                    fileName = "Error";
362
                }
363
            }
364
        }else{
365
            List<Classification> classificationList = classificationService.list(null, 1, null
366
                    , null, null);
367
            if (!classificationList.isEmpty()){
368
                fileName = classificationList.get(0).getTitleCache();
369
            }else{
370
               //handle via repso
371
                response.sendError(404, "No classification found");
372
                fileName = "Error";
373
            }
374
        }
375

  
376

  
377
        fileName = fileName + "_" + uuidListToString(subtreeUuids, 40) + ".zip";
378

  
379
        return fileName;
380

  
381
    }
382

  
383
    private String uuidListToString(UuidList uuidList, Integer truncate) {
384
        String result = null;
385
        for (UUID uuid : uuidList){
386
            result = CdmUtils.concat("_", uuid.toString());
387
        }
388
        if (result != null && result.length() > truncate){
389
            result = result.substring(0, truncate);
390
        }
391
        return result;
392
    }
393

  
394
    @Override
395
    public void setService(IService service) {}
396

  
397
    @Override
398
    public void setResourceLoader(ResourceLoader resourceLoader) {
399
        this.resourceLoader = resourceLoader;
400
    }
401

  
402
}
src/site/apt/remote/dwca-tax-export-default.apt
1
                                   --------------------------
2
                                    DWCA TAXON EXPORT API
3
                                    --------------------------
4

  
5
{CDM DWCA TAXON API}
6

  
7
    TODO
8

  
9
	
10
*	The request parameters are :	
11
	
12
	
13

  

Also available in: Unified diff

Add picture from clipboard (Maximum size: 40 MB)