1
|
package org.cybertaxonomy.utis.refine;
|
2
|
|
3
|
import java.io.IOException;
|
4
|
import java.io.InputStream;
|
5
|
import java.io.StringWriter;
|
6
|
import java.net.URI;
|
7
|
import java.net.URISyntaxException;
|
8
|
import java.util.ArrayList;
|
9
|
import java.util.Iterator;
|
10
|
import java.util.List;
|
11
|
|
12
|
import org.apache.commons.io.IOUtils;
|
13
|
import org.apache.http.Header;
|
14
|
import org.apache.http.HttpEntity;
|
15
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
16
|
import org.apache.http.client.methods.HttpPost;
|
17
|
import org.apache.http.client.utils.URIBuilder;
|
18
|
import org.apache.http.entity.ByteArrayEntity;
|
19
|
import org.apache.http.entity.mime.MultipartEntityBuilder;
|
20
|
import org.apache.http.impl.client.CloseableHttpClient;
|
21
|
import org.apache.http.impl.client.HttpClients;
|
22
|
import org.apache.http.message.BasicHeader;
|
23
|
import org.apache.http.util.EntityUtils;
|
24
|
import org.cybertaxonomy.utis.checklist.DRFChecklistException;
|
25
|
import org.cybertaxonomy.utis.utils.JSONUtils;
|
26
|
import org.json.simple.JSONObject;
|
27
|
|
28
|
public class RefineClient {
|
29
|
private static String boundary = "----dcBoundary";
|
30
|
|
31
|
private String server;
|
32
|
private String port;
|
33
|
|
34
|
public RefineClient(String server, String port) {
|
35
|
this.server = server;
|
36
|
this.port=port;
|
37
|
}
|
38
|
|
39
|
public String initJob() throws IOException, DRFChecklistException, URISyntaxException {
|
40
|
String jobID = null;
|
41
|
List<Header> headers = new ArrayList<Header>();
|
42
|
headers.add(new BasicHeader("Content-Type","application/http"));
|
43
|
headers.add(new BasicHeader("Accept","application/json"));
|
44
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/create-importing-job");
|
45
|
|
46
|
String initResponse = callCoreRefineService(uriBuilder.build(),headers,null);
|
47
|
|
48
|
JSONObject jsonResponse = (JSONObject) JSONUtils.parseJsonToObject(initResponse);
|
49
|
Long jobIDLong = (Long)jsonResponse.get("jobID");
|
50
|
if(jobIDLong != null) {
|
51
|
jobID = String.valueOf(jobIDLong);
|
52
|
}
|
53
|
return jobID;
|
54
|
}
|
55
|
|
56
|
public String uploadData(String csvData, String jobID) throws RefineException, IOException, URISyntaxException {
|
57
|
String uploadResponse = "";
|
58
|
|
59
|
if(csvData != null && !csvData.isEmpty()) {
|
60
|
MultipartEntityBuilder meBuilder = getMEBuilder();
|
61
|
meBuilder.setBoundary(boundary);
|
62
|
|
63
|
String lineEnding = "\r\n";
|
64
|
String beginStr = "--" + boundary + lineEnding +
|
65
|
"Content-Disposition: form-data; name=\"upload\"; filename=\"dcinput.txt\"" + lineEnding +
|
66
|
"Content-Type: text/plain" + lineEnding + lineEnding;
|
67
|
String endStr = lineEnding + "--" + boundary + "--";
|
68
|
//String inputBody = beginStr + csvData.replaceAll(System.getProperty("line.separator"),"\r\n") + endStr;
|
69
|
String inputBody = beginStr + csvData + endStr;
|
70
|
|
71
|
meBuilder.addBinaryBody("input", inputBody.getBytes());
|
72
|
|
73
|
List<Header> headers = new ArrayList<Header>();
|
74
|
headers.add(new BasicHeader("Content-Type","multipart/form-data; boundary=" + boundary));
|
75
|
headers.add(new BasicHeader("Accept","text/html"));
|
76
|
|
77
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/importing-controller");
|
78
|
uriBuilder.setParameter("jobID", jobID);
|
79
|
uriBuilder.setParameter("subCommand","load-raw-data");
|
80
|
uriBuilder.setParameter("controller","core/default-importing-controller");
|
81
|
|
82
|
uploadResponse = callCoreRefineService(uriBuilder.build(),headers,new ByteArrayEntity(inputBody.getBytes()));
|
83
|
|
84
|
} else {
|
85
|
throw new RefineException("Data to be uploaded to Refine server is null or empty");
|
86
|
}
|
87
|
return uploadResponse;
|
88
|
}
|
89
|
|
90
|
public boolean checkImportJobStatus(String jobID) throws IOException, URISyntaxException, DRFChecklistException {
|
91
|
boolean importDone = false;
|
92
|
String checkResponse;
|
93
|
List<Header> headers = new ArrayList<Header>();
|
94
|
headers.add(new BasicHeader("Content-Type","multipart/form-data"));
|
95
|
headers.add(new BasicHeader("Accept","application/json"));
|
96
|
|
97
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/get-importing-job-status");
|
98
|
uriBuilder.setParameter("jobID", jobID);
|
99
|
while(!importDone) {
|
100
|
checkResponse = callCoreRefineService(uriBuilder.build(),headers,null);
|
101
|
System.out.println("importJobCheck : " + checkResponse);
|
102
|
JSONObject jsonResponse = (JSONObject) JSONUtils.parseJsonToObject(checkResponse);
|
103
|
JSONObject job = (JSONObject)jsonResponse.get("job");
|
104
|
JSONObject config = (JSONObject)job.get("config");
|
105
|
boolean hasData = (Boolean)config.get("hasData");
|
106
|
String state = (String)config.get("state");
|
107
|
if(hasData && state.equals("ready"))
|
108
|
importDone = true;
|
109
|
}
|
110
|
return importDone;
|
111
|
}
|
112
|
|
113
|
public String initializeParser(String jobID) throws IOException, URISyntaxException {
|
114
|
|
115
|
List<Header> headers = new ArrayList<Header>();
|
116
|
headers.add(new BasicHeader("Content-Type","application/xml"));
|
117
|
headers.add(new BasicHeader("Accept","application/json"));
|
118
|
|
119
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/importing-controller");
|
120
|
uriBuilder.setParameter("controller","core/default-importing-controller");
|
121
|
uriBuilder.setParameter("jobID", jobID);
|
122
|
uriBuilder.setParameter("subCommand","initialize-parser-ui");
|
123
|
uriBuilder.setParameter("format","text/line-based/*sv");
|
124
|
|
125
|
return callCoreRefineService(uriBuilder.build(),headers,null);
|
126
|
}
|
127
|
|
128
|
public String updateFormat(String jobID) throws IOException, URISyntaxException {
|
129
|
|
130
|
List<Header> headers = new ArrayList<Header>();
|
131
|
headers.add(new BasicHeader("Content-Type","application/x-www-form-urlencoded"));
|
132
|
headers.add(new BasicHeader("Accept","application/json"));
|
133
|
|
134
|
String options = "options={\"encoding\":\"\",\"separator\":\",\",\"ignoreLines\":-1,\"headerLines\":1,\"skipDataLines\":0,\"limit\":-1,\"storeBlankRows\":false,\"guessCellValueTypes\":true,\"processQuotes\":true,\"storeBlankCellsAsNulls\":true,\"includeFileSources\":false,\"projectName\":\"" + jobID + "\"}";
|
135
|
|
136
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/importing-controller");
|
137
|
uriBuilder.setParameter("controller","core/default-importing-controller");
|
138
|
uriBuilder.setParameter("jobID", jobID);
|
139
|
uriBuilder.setParameter("subCommand","update-format-and-options");
|
140
|
uriBuilder.setParameter("format","text/line-based/*sv");
|
141
|
|
142
|
return callCoreRefineService(uriBuilder.build(),headers,new ByteArrayEntity(options.getBytes()));
|
143
|
}
|
144
|
|
145
|
public String createProject(String jobID) throws IOException, URISyntaxException {
|
146
|
|
147
|
List<Header> headers = new ArrayList<Header>();
|
148
|
headers.add(new BasicHeader("Content-Type","application/x-www-form-urlencoded"));
|
149
|
headers.add(new BasicHeader("Accept","application/json"));
|
150
|
|
151
|
String options = "options={\"encoding\":\"\",\"separator\":\",\",\"ignoreLines\":-1,\"headerLines\":1,\"skipDataLines\":0,\"limit\":-1,\"storeBlankRows\":false,\"guessCellValueTypes\":true,\"processQuotes\":true,\"storeBlankCellsAsNulls\":true,\"includeFileSources\":false,\"projectName\":\"" + jobID + "\"}";
|
152
|
|
153
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/importing-controller");
|
154
|
uriBuilder.setParameter("controller","core/default-importing-controller");
|
155
|
uriBuilder.setParameter("jobID", jobID);
|
156
|
uriBuilder.setParameter("subCommand","create-project");
|
157
|
uriBuilder.setParameter("format","text/line-based/*sv");
|
158
|
|
159
|
|
160
|
return callCoreRefineService(uriBuilder.build(),headers,new ByteArrayEntity(options.getBytes()));
|
161
|
}
|
162
|
|
163
|
public String checkCreateProjectStatus(String jobID) throws IOException, URISyntaxException, DRFChecklistException, InterruptedException {
|
164
|
boolean createDone = false;
|
165
|
String checkResponse;
|
166
|
List<Header> headers = new ArrayList<Header>();
|
167
|
headers.add(new BasicHeader("Content-Type","application/xml"));
|
168
|
headers.add(new BasicHeader("Accept","application/json"));
|
169
|
|
170
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/get-importing-job-status");
|
171
|
uriBuilder.setParameter("jobID", jobID);
|
172
|
|
173
|
Long percent;
|
174
|
String projectID = null;
|
175
|
while(!createDone) {
|
176
|
checkResponse = callCoreRefineService(uriBuilder.build(),headers,null);
|
177
|
System.out.println("createProjectCheck : " + checkResponse);
|
178
|
JSONObject jsonResponse = (JSONObject) JSONUtils.parseJsonToObject(checkResponse);
|
179
|
JSONObject job = (JSONObject)jsonResponse.get("job");
|
180
|
JSONObject config = (JSONObject)job.get("config");
|
181
|
JSONObject progress = (JSONObject)config.get("progress");
|
182
|
if(progress != null) {
|
183
|
percent = (Long)progress.get("percent");
|
184
|
Long projectIDLong = (Long)config.get("projectID");
|
185
|
projectID = String.valueOf(projectIDLong);
|
186
|
createDone = (percent == 100);
|
187
|
}
|
188
|
Thread.sleep(1000);
|
189
|
}
|
190
|
|
191
|
return projectID;
|
192
|
}
|
193
|
|
194
|
public String deleteProject(String projectID) throws IOException, URISyntaxException {
|
195
|
|
196
|
List<Header> headers = new ArrayList<Header>();
|
197
|
headers.add(new BasicHeader("Content-Type","application/x-www-form-urlencoded"));
|
198
|
headers.add(new BasicHeader("Accept","application/json"));
|
199
|
|
200
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/delete-project");
|
201
|
uriBuilder.setParameter("project", projectID);
|
202
|
|
203
|
return callCoreRefineService(uriBuilder.build(),headers,null);
|
204
|
|
205
|
}
|
206
|
|
207
|
public String exportData(String projectID) throws RefineException, IOException, URISyntaxException {
|
208
|
List<Header> headers = new ArrayList<Header>();
|
209
|
headers.add(new BasicHeader("Content-Type","application/x-www-form-urlencoded"));
|
210
|
headers.add(new BasicHeader("Accept","text/html"));
|
211
|
URIBuilder uriBuilder = getURIBuilder(server,port,"/command/core/export-rows/" + projectID + ".csv");
|
212
|
|
213
|
uriBuilder.setParameter("project", projectID);
|
214
|
uriBuilder.setParameter("engine","{\"facets\":[],\"mode\":\"row-based\"}");
|
215
|
uriBuilder.setParameter("format","csv");
|
216
|
return callCoreRefineService(uriBuilder.build(),headers,null);
|
217
|
}
|
218
|
|
219
|
public String callCoreRefineService(URI uri,
|
220
|
List<Header> headers,
|
221
|
HttpEntity reqEntity) throws IOException {
|
222
|
|
223
|
|
224
|
String responseString = null;
|
225
|
CloseableHttpClient httpClient = HttpClients.createDefault();
|
226
|
|
227
|
try {
|
228
|
HttpPost httpPost = new HttpPost(uri);
|
229
|
|
230
|
Iterator<Header> headerItr = headers.iterator();
|
231
|
while(headerItr.hasNext()) {
|
232
|
httpPost.addHeader(headerItr.next());
|
233
|
}
|
234
|
if(reqEntity != null) {
|
235
|
httpPost.setEntity(reqEntity);
|
236
|
}
|
237
|
|
238
|
System.out.println("executing request " + httpPost.getRequestLine());
|
239
|
CloseableHttpResponse response = httpClient.execute(httpPost);
|
240
|
|
241
|
try {
|
242
|
System.out.println("----------------------------------------");
|
243
|
System.out.println(response.getStatusLine());
|
244
|
System.out.println(response.toString());
|
245
|
HttpEntity resEntity = response.getEntity();
|
246
|
if (resEntity != null) {
|
247
|
StringWriter writer = new StringWriter();
|
248
|
responseString = contentToString(resEntity.getContent());
|
249
|
System.out.println(responseString);
|
250
|
System.out.println("Response content length: " + resEntity.getContentLength());
|
251
|
}
|
252
|
EntityUtils.consume(resEntity);
|
253
|
} finally {
|
254
|
response.close();
|
255
|
}
|
256
|
} finally {
|
257
|
httpClient.close();
|
258
|
}
|
259
|
return responseString;
|
260
|
}
|
261
|
|
262
|
public MultipartEntityBuilder getMEBuilder() {
|
263
|
MultipartEntityBuilder meBuilder = MultipartEntityBuilder.create();
|
264
|
|
265
|
return meBuilder;
|
266
|
}
|
267
|
|
268
|
private String contentToString(InputStream content) throws IOException {
|
269
|
StringWriter writer = new StringWriter();
|
270
|
IOUtils.copy(content, writer, "UTF-8");
|
271
|
return writer.toString();
|
272
|
}
|
273
|
|
274
|
private URIBuilder getURIBuilder(String host, String port, String path) {
|
275
|
URIBuilder uriBuilder = new URIBuilder();
|
276
|
uriBuilder.setScheme("http");
|
277
|
uriBuilder.setHost(host);
|
278
|
uriBuilder.setPort(Integer.valueOf(port));
|
279
|
uriBuilder.setPath(path);
|
280
|
|
281
|
return uriBuilder;
|
282
|
}
|
283
|
|
284
|
|
285
|
}
|