Revision 1a34bf3a
Added by Andreas Kohlbecker over 8 years ago
pom.xml | ||
---|---|---|
16 | 16 |
<id>onejar-maven-plugin.googlecode.com</id> |
17 | 17 |
<url>http://onejar-maven-plugin.googlecode.com/svn/mavenrepo</url> |
18 | 18 |
</repository> |
19 |
<repository> |
|
20 |
<id>EditRepository</id> |
|
21 |
<url>http://dev.e-taxonomy.eu/mavenrepo/</url> |
|
22 |
</repository> |
|
23 |
<!-- codehaus mule repository needed for yourkit --> |
|
24 |
<repository> |
|
25 |
<id>CodehausMuleRepository</id> |
|
26 |
<url>http://dist.codehaus.org/mule/dependencies/maven2/</url> |
|
27 |
</repository> |
|
19 | 28 |
</repositories> |
20 | 29 |
|
21 | 30 |
<properties> |
... | ... | |
24 | 33 |
<apache.jena.version>3.0.0</apache.jena.version> |
25 | 34 |
<tinkerpop.version>2.6.0</tinkerpop.version> |
26 | 35 |
<sesame.version>2.7.10</sesame.version><!-- should match the version as used in tinkerpop.blueprints --> |
36 |
<neo4j.version>1.9.9</neo4j.version> |
|
27 | 37 |
</properties> |
28 | 38 |
|
29 | 39 |
<dependencies> |
... | ... | |
165 | 175 |
<artifactId>sesame-repository-sail</artifactId> |
166 | 176 |
<version>${sesame.version}</version> |
167 | 177 |
</dependency> |
178 |
<dependency> |
|
179 |
<groupId>org.neo4j</groupId> |
|
180 |
<artifactId>neo4j</artifactId> |
|
181 |
<version>${neo4j.version}</version> |
|
182 |
</dependency> |
|
183 |
<dependency> |
|
184 |
<groupId>org.neo4j</groupId> |
|
185 |
<artifactId>neo4j-ha</artifactId> |
|
186 |
<version>${neo4j.version}</version> |
|
187 |
</dependency> |
|
188 |
<!-- yourkit profiler api controller --> |
|
189 |
<!-- the 2015 version is not available via maven |
|
190 |
<dependency> |
|
191 |
<groupId>com.yourkit</groupId> |
|
192 |
<artifactId>yjp-controller-api-redist</artifactId> |
|
193 |
<version>9.0.8</version> |
|
194 |
<!- - <version>10.0.6</version> TODO find repo or put into our own - -> |
|
195 |
<scope>test</scope> |
|
196 |
</dependency> |
|
197 |
--> |
|
168 | 198 |
</dependencies> |
169 | 199 |
|
170 | 200 |
<!-- === THIS IS ONLY NEEDED FOR DEPLOYMENT TO TAVERNA === <pluginRepositories> |
src/main/java/org/bgbm/biovel/drf/checklist/EEA_BDC_Client.java | ||
---|---|---|
1 | 1 |
package org.bgbm.biovel.drf.checklist; |
2 | 2 |
|
3 |
import java.io.PrintStream; |
|
3 | 4 |
import java.net.URI; |
5 |
import java.util.ArrayList; |
|
6 |
import java.util.Collection; |
|
4 | 7 |
import java.util.EnumSet; |
8 |
import java.util.Iterator; |
|
5 | 9 |
import java.util.List; |
6 |
import java.util.NoSuchElementException; |
|
7 | 10 |
|
8 | 11 |
import org.apache.jena.rdf.model.Model; |
9 |
import org.apache.jena.rdf.model.Property; |
|
10 |
import org.apache.jena.rdf.model.ResIterator; |
|
11 | 12 |
import org.apache.jena.rdf.model.Resource; |
12 |
import org.apache.jena.rdf.model.StmtIterator; |
|
13 | 13 |
import org.bgbm.biovel.drf.client.ServiceProviderInfo; |
14 | 14 |
import org.bgbm.biovel.drf.query.IQueryClient; |
15 | 15 |
import org.bgbm.biovel.drf.query.SparqlClient; |
16 | 16 |
import org.bgbm.biovel.drf.query.TinkerPopClient; |
17 | 17 |
import org.bgbm.biovel.drf.store.Neo4jStore; |
18 |
import org.bgbm.biovel.drf.store.Store; |
|
18 | 19 |
import org.bgbm.biovel.drf.store.TDBStore; |
19 |
import org.bgbm.biovel.drf.tnr.msg.Classification; |
|
20 | 20 |
import org.bgbm.biovel.drf.tnr.msg.NameType; |
21 | 21 |
import org.bgbm.biovel.drf.tnr.msg.Query; |
22 | 22 |
import org.bgbm.biovel.drf.tnr.msg.Query.Request; |
... | ... | |
28 | 28 |
import org.bgbm.biovel.drf.tnr.msg.TaxonName; |
29 | 29 |
import org.bgbm.biovel.drf.tnr.msg.TnrMsg; |
30 | 30 |
import org.bgbm.biovel.drf.utils.IdentifierUtils; |
31 |
import org.bgbm.biovel.drf.utils.Profiler; |
|
31 | 32 |
import org.bgbm.biovel.drf.utils.TnrMsgUtils; |
33 |
import org.neo4j.graphdb.Relationship; |
|
34 |
import org.openrdf.query.MalformedQueryException; |
|
35 |
import org.openrdf.query.QueryEvaluationException; |
|
36 |
import org.openrdf.query.QueryLanguage; |
|
37 |
import org.openrdf.query.TupleQuery; |
|
38 |
import org.openrdf.query.TupleQueryResult; |
|
39 |
import org.openrdf.repository.RepositoryException; |
|
40 |
import org.openrdf.repository.sail.SailRepositoryConnection; |
|
41 |
|
|
42 |
import com.tinkerpop.blueprints.CloseableIterable; |
|
43 |
import com.tinkerpop.blueprints.Direction; |
|
44 |
import com.tinkerpop.blueprints.Graph; |
|
45 |
import com.tinkerpop.blueprints.Index; |
|
46 |
import com.tinkerpop.blueprints.Vertex; |
|
47 |
import com.tinkerpop.blueprints.impls.neo4j.Neo4jGraph; |
|
48 |
import com.tinkerpop.blueprints.impls.neo4j.Neo4jVertex; |
|
49 |
import com.tinkerpop.blueprints.oupls.sail.GraphSail; |
|
50 |
import com.tinkerpop.gremlin.java.GremlinPipeline; |
|
51 |
import com.tinkerpop.pipes.PipeFunction; |
|
52 |
import com.tinkerpop.pipes.util.FastNoSuchElementException; |
|
32 | 53 |
|
33 | 54 |
public class EEA_BDC_Client extends AggregateChecklistClient<TinkerPopClient> { |
34 | 55 |
|
... | ... | |
39 | 60 |
public static final String LABEL = "European Environment Agency (EEA) Biodiversity data centre (BDC)"; |
40 | 61 |
public static final String DOC_URL = "http://semantic.eea.europa.eu/documentation"; |
41 | 62 |
public static final String COPYRIGHT_URL = "http://www.eea.europa.eu/legal/eea-data-policy"; |
63 |
|
|
42 | 64 |
private static final String SPARQL_ENDPOINT_URL = "http://semantic.eea.europa.eu/sparql"; |
43 |
private static final String RDF_FILE_URL = "http://localhost/download/species.rdf.gz"; // http://eunis.eea.europa.eu/rdf/species.rdf.gz |
|
44 | 65 |
private static final boolean USE_REMOTE_SERVICE = false; |
66 |
|
|
67 |
private static final String SPECIES_RDF_FILE_URL = "http://localhost/download/species.rdf.gz"; // http://eunis.eea.europa.eu/rdf/species.rdf.gz |
|
68 |
private static final String LEGALREFS_RDF_FILE_URL = "http://localhost/download/legalrefs.rdf.gz"; // http://eunis.eea.europa.eu/rdf/legalrefs.rdf.gz |
|
69 |
private static final String REFERENCES_RDF_FILE_URL = "http://localhost/download/references.rdf.gz"; // http://eunis.eea.europa.eu/rdf/references.rdf.gz |
|
45 | 70 |
private static final boolean REFRESH_TDB = false; |
71 |
|
|
46 | 72 |
private static final Class<? extends IQueryClient> clientClass = TinkerPopClient.class; |
47 | 73 |
|
48 | 74 |
private static final int MAX_PAGING_LIMIT = 50; |
... | ... | |
76 | 102 |
DWC("dwc", "http://rs.tdwg.org/dwc/terms/"), |
77 | 103 |
RDF("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"), |
78 | 104 |
RDFS("rdfs", "http://www.w3.org/2000/01/rdf-schema#"), |
79 |
SKOS_CORE("scos_core", "http://www.w3.org/2004/02/skos/core#"); |
|
105 |
SKOS_CORE("scos_core", "http://www.w3.org/2004/02/skos/core#"), |
|
106 |
DC("dc", "http://purl.org/dc/terms/source"), |
|
107 |
DCTERMS("dcterms", "http://purl.org/dc/terms/"); |
|
80 | 108 |
|
81 | 109 |
private String schemaUri; |
82 | 110 |
private String abbreviation; |
... | ... | |
95 | 123 |
return abbreviation; |
96 | 124 |
} |
97 | 125 |
|
126 |
public String propertyURI(String name) { |
|
127 |
return schemaUri + name; |
|
128 |
} |
|
129 |
|
|
98 | 130 |
} |
99 | 131 |
|
100 | 132 |
public enum SubCheckListId { |
... | ... | |
132 | 164 |
throw new RuntimeException("Creation of TripleStore failed", e1); |
133 | 165 |
} |
134 | 166 |
if(REFRESH_TDB) { |
135 |
// use downloadable rdf |
|
136 |
try { |
|
137 |
tripleStore.loadIntoStore(RDF_FILE_URL); |
|
138 |
} catch (Exception e) { |
|
139 |
logger.error("Loading " + RDF_FILE_URL + " into TripleStore failed", e); |
|
140 |
} |
|
167 |
updateStore(tripleStore); |
|
141 | 168 |
} |
142 | 169 |
//FIXME queryClient = new SparqlClient(tripleStore); |
143 | 170 |
|
... | ... | |
153 | 180 |
throw new RuntimeException("Creation of Neo4jStore failed", e1); |
154 | 181 |
} |
155 | 182 |
if(REFRESH_TDB) { |
156 |
// use downloadable rdf |
|
157 |
try { |
|
158 |
neo4jStore.loadIntoStore(RDF_FILE_URL); |
|
159 |
} catch (Exception e) { |
|
160 |
throw new RuntimeException("Loading " + RDF_FILE_URL + " into Neo4jStore failed", e); |
|
161 |
} |
|
183 |
updateStore(neo4jStore); |
|
162 | 184 |
} |
163 | 185 |
queryClient = new TinkerPopClient(neo4jStore); |
164 | 186 |
|
... | ... | |
169 | 191 |
} |
170 | 192 |
} |
171 | 193 |
|
194 |
/** |
|
195 |
* @param neo4jStore |
|
196 |
*/ |
|
197 |
private void updateStore(Store neo4jStore) { |
|
198 |
try { |
|
199 |
neo4jStore.loadIntoStore( |
|
200 |
//SPECIES_RDF_FILE_URL, |
|
201 |
LEGALREFS_RDF_FILE_URL, |
|
202 |
REFERENCES_RDF_FILE_URL |
|
203 |
); |
|
204 |
} catch (Exception e) { |
|
205 |
throw new RuntimeException("Loading " |
|
206 |
+ SPECIES_RDF_FILE_URL + ", " |
|
207 |
+ LEGALREFS_RDF_FILE_URL + ", " |
|
208 |
+ REFERENCES_RDF_FILE_URL + |
|
209 |
" into Neo4jStore failed", e); |
|
210 |
} |
|
211 |
} |
|
212 |
|
|
172 | 213 |
@Override |
173 | 214 |
public ServiceProviderInfo buildServiceProviderInfo() { |
174 | 215 |
|
... | ... | |
203 | 244 |
return queryString; |
204 | 245 |
} |
205 | 246 |
|
206 |
private Taxon createTaxon(Model model, Resource taxonR) {
|
|
247 |
private Taxon createTaxon(Vertex v) {
|
|
207 | 248 |
|
208 | 249 |
Taxon taxon = new Taxon(); |
209 | 250 |
|
210 |
TaxonName taxonName = createTaxonName(taxonR);
|
|
251 |
TaxonName taxonName = createTaxonName(v);
|
|
211 | 252 |
|
212 | 253 |
// Taxon |
213 | 254 |
taxon.setTaxonName(taxonName); |
214 |
taxon.setIdentifier(taxonR.getURI());
|
|
215 |
taxon.setAccordingTo(queryClient.objectAsString(taxonR, RdfSchema.DWC, "nameAccordingToID"));
|
|
216 |
URI typeUri = queryClient.objectAsURI(taxonR, RdfSchema.RDF, "type");
|
|
255 |
taxon.setIdentifier(v.getId().toString());
|
|
256 |
taxon.setAccordingTo(queryClient.relatedVertexValue(v, RdfSchema.DWC, "nameAccordingToID"));
|
|
257 |
URI typeUri = queryClient.relatedVertexURI(v, RdfSchema.RDF, "type");
|
|
217 | 258 |
taxon.setTaxonomicStatus(typeUri.getFragment()); |
218 | 259 |
|
219 |
createSources(model, taxonR, taxon); |
|
260 |
createSources(v, taxon); |
|
261 |
|
|
262 |
/* |
|
220 | 263 |
|
221 | 264 |
// classification |
222 | 265 |
Classification c = null; |
... | ... | |
269 | 312 |
if(c != null) { |
270 | 313 |
taxon.setClassification(c); |
271 | 314 |
} |
315 |
*/ |
|
272 | 316 |
return taxon; |
273 | 317 |
} |
274 | 318 |
|
... | ... | |
277 | 321 |
* @param taxonR |
278 | 322 |
* @param taxonBase |
279 | 323 |
*/ |
280 |
private void createSources(Model model, Resource taxonR, TaxonBase taxonBase) { |
|
324 |
private void createSources(Vertex v, TaxonBase taxonBase) { |
|
325 |
|
|
281 | 326 |
// Sources are source references, re there others like data bases? |
282 |
for ( StmtIterator refIt = taxonR.listProperties(model.getProperty(RdfSchema.EUNIS_SPECIES.schemaUri, "hasLegalReference")); refIt.hasNext();) { |
|
283 |
try { |
|
284 |
Source source = new Source(); |
|
285 |
Resource sourceR = refIt.next().getObject().asResource(); |
|
286 |
String sourceName = queryClient.objectAsString(sourceR, RdfSchema.RDFS, "source"); |
|
287 |
source.setName(sourceName); |
|
288 |
taxonBase.getSources().add(source); |
|
289 |
} catch (NoSuchElementException e) { |
|
290 |
logger.debug("No statements for rdf:hasLegalReference" , e); |
|
327 |
|
|
328 |
GremlinPipeline<Graph, Vertex> taxonPipe = new GremlinPipeline<Graph, Vertex>(v); |
|
329 |
|
|
330 |
try { |
|
331 |
List<Vertex> titleVs = taxonPipe |
|
332 |
.outE(RdfSchema.EUNIS_SPECIES.propertyURI("hasLegalReference")).inV() |
|
333 |
.outE(RdfSchema.DCTERMS.propertyURI("source")).inV().dedup() |
|
334 |
.outE(RdfSchema.DCTERMS.propertyURI("title")).inV() |
|
335 |
.toList(); |
|
336 |
for(Vertex tv : titleVs) { |
|
337 |
Source source = new Source(); |
|
338 |
logger.error(tv.toString()); |
|
339 |
source.setName(tv.getProperty(GraphSail.VALUE).toString()); |
|
340 |
taxonBase.getSources().add(source); |
|
291 | 341 |
} |
342 |
} catch (FastNoSuchElementException e) { |
|
343 |
logger.debug("No sources found"); |
|
292 | 344 |
} |
293 | 345 |
} |
294 | 346 |
|
... | ... | |
296 | 348 |
* @param taxonR |
297 | 349 |
* @return |
298 | 350 |
*/ |
299 |
private TaxonName createTaxonName(Resource taxonR) { |
|
351 |
private TaxonName createTaxonName(Vertex v) { |
|
352 |
|
|
300 | 353 |
TaxonName taxonName = new TaxonName(); |
301 | 354 |
// TaxonName |
302 |
taxonName.setFullName(queryClient.objectAsString(taxonR, RdfSchema.RDFS, "label"));
|
|
355 |
taxonName.setFullName(queryClient.relatedVertexValue(v, RdfSchema.RDFS, "label"));
|
|
303 | 356 |
// TODO rename CanonicalName to scientificName? compare with dwc:scientificName |
304 |
taxonName.setCanonicalName(queryClient.objectAsString(taxonR, RdfSchema.EUNIS_SPECIES, "binomialName"));
|
|
305 |
taxonName.setRank(queryClient.objectAsString(taxonR, RdfSchema.EUNIS_SPECIES, "taxonomicRank"));
|
|
357 |
taxonName.setCanonicalName(queryClient.relatedVertexValue(v, RdfSchema.EUNIS_SPECIES, "binomialName"));
|
|
358 |
taxonName.setRank(queryClient.relatedVertexValue(v, RdfSchema.EUNIS_SPECIES, "taxonomicRank"));
|
|
306 | 359 |
return taxonName; |
307 | 360 |
} |
308 | 361 |
|
309 | 362 |
|
363 |
private void createSynonyms(Vertex taxonV, Response tnrResponse) { |
|
310 | 364 |
|
311 | 365 |
|
312 |
private void createSynonyms(Resource taxonR, Response tnrResponse) { |
|
313 |
|
|
314 |
List<Resource> synonymRList = queryForSynonyms(taxonR); |
|
315 |
|
|
316 |
for (Resource synonymR : synonymRList) { |
|
317 |
|
|
318 |
URI typeUri = queryClient.objectAsURI(synonymR, RdfSchema.RDF, "type"); |
|
319 |
String status = typeUri.getFragment(); |
|
366 |
GremlinPipeline<Graph, Vertex> taxonPipe = new GremlinPipeline<Graph, Vertex>(taxonV); |
|
320 | 367 |
|
368 |
try { |
|
369 |
List<Vertex> synonymVs = taxonPipe |
|
370 |
.inE(RdfSchema.EUNIS_SPECIES.propertyURI("eunisPrimaryName")).outV().dedup() |
|
371 |
.toList(); |
|
372 |
for(Vertex synonymV : synonymVs) { |
|
373 |
String typeUri = queryClient.relatedVertexValue(synonymV, RdfSchema.RDF, "type"); |
|
374 |
String status = null; |
|
375 |
try { |
|
376 |
status = URI.create(typeUri).getFragment(); |
|
377 |
} catch (Exception e) { |
|
321 | 378 |
|
322 |
if (status != null && status.equals("SpeciesSynonym")) {
|
|
379 |
}
|
|
323 | 380 |
|
324 |
Synonym synonym = new Synonym();
|
|
381 |
if (status != null && status.equals("SpeciesSynonym")) {
|
|
325 | 382 |
|
326 |
TaxonName taxonName = createTaxonName(synonymR);
|
|
383 |
Synonym synonym = new Synonym();
|
|
327 | 384 |
|
328 |
synonym.setTaxonomicStatus(status); |
|
329 |
synonym.setTaxonName(taxonName); |
|
330 |
synonym.setAccordingTo(queryClient.objectAsString(synonymR, RdfSchema.DWC, "nameAccordingToID")); |
|
385 |
TaxonName taxonName = createTaxonName(synonymV); |
|
386 |
synonym.setTaxonomicStatus(status); |
|
387 |
synonym.setTaxonName(taxonName); |
|
388 |
synonym.setAccordingTo(queryClient.relatedVertexValue(synonymV, RdfSchema.DWC, "nameAccordingToID")); |
|
331 | 389 |
|
332 |
createSources(synonymR.getModel(), synonymR, synonym);
|
|
390 |
createSources(synonymV, synonym);
|
|
333 | 391 |
|
334 |
tnrResponse.getSynonym().add(synonym); |
|
392 |
tnrResponse.getSynonym().add(synonym); |
|
393 |
} |
|
335 | 394 |
} |
395 |
} catch (FastNoSuchElementException e) { |
|
396 |
logger.debug("No sources found"); |
|
336 | 397 |
} |
398 |
|
|
337 | 399 |
} |
338 | 400 |
|
339 | 401 |
/** |
... | ... | |
374 | 436 |
*/ |
375 | 437 |
private List<Resource> listSynonymResources(Model model, Resource taxonR) { |
376 | 438 |
List<Resource> synonymRList; |
439 |
/* |
|
377 | 440 |
Property filterProperty = model.createProperty(RdfSchema.EUNIS_SPECIES.schemaUri, "eunisPrimaryName"); |
378 | 441 |
synonymRList = queryClient.listResources(model, filterProperty, null, taxonR); |
379 | 442 |
return synonymRList; |
443 |
*/ |
|
444 |
return null; |
|
380 | 445 |
} |
381 | 446 |
|
382 | 447 |
@Override |
... | ... | |
391 | 456 |
for (ServiceProviderInfo checklistInfo : getServiceProviderInfo().getSubChecklists()) { |
392 | 457 |
|
393 | 458 |
Query query = singleQueryFrom(tnrMsg); |
394 |
StringBuilder queryString = prepareQueryString(); |
|
459 |
|
|
460 |
boolean TUPLEQUERY = false; |
|
461 |
boolean Neo4jINDEX = true; |
|
395 | 462 |
|
396 | 463 |
String filter; |
464 |
String queryString = query.getRequest().getQueryString(); |
|
465 |
logger.debug("queryString: "+ queryString); |
|
466 |
PipeFunction<Vertex, Boolean> matchFilter; |
|
397 | 467 |
if(query.getRequest().getSearchMode().equals(SearchMode.scientificNameLike.name())) { |
398 |
filter = "(regex(?name, \"^" + query.getRequest().getQueryString() + "\"))"; |
|
468 |
filter = "(regex(?name, \"^" + queryString + "\"))"; |
|
469 |
matchFilter = queryClient.createStarttWithFilter(queryString); |
|
470 |
queryString = queryString + "*"; |
|
399 | 471 |
} else { |
400 |
filter = "(?name = \"" + query.getRequest().getQueryString() + "\")"; |
|
472 |
filter = "(?name = \"" + queryString + "\")"; |
|
473 |
matchFilter = queryClient.createEqualsFilter(queryString); |
|
401 | 474 |
} |
402 | 475 |
|
403 |
queryString.append( |
|
404 |
"DESCRIBE ?eunisurl \n" |
|
405 |
+ "WHERE {\n" |
|
406 |
+ " ?eunisurl es:binomialName ?name . \n" |
|
407 |
+ " FILTER " + filter + " \n" |
|
408 |
+ "} \n" |
|
409 |
+ "LIMIT " + MAX_PAGING_LIMIT + " OFFSET 0" |
|
410 |
); |
|
476 |
if(TUPLEQUERY) { |
|
477 |
StringBuilder sparql = prepareQueryString(); |
|
478 |
sparql.append( |
|
479 |
"SELECT ?eunisurl \n" |
|
480 |
+ "WHERE {\n" |
|
481 |
+ " ?eunisurl es:binomialName ?name . \n" |
|
482 |
+ " FILTER " + filter + " \n" |
|
483 |
+ "}" |
|
484 |
); |
|
485 |
|
|
486 |
Neo4jGraph neo4jGraph = (Neo4jGraph)queryClient.graph(); |
|
487 |
Vertex v = neo4jGraph.getVertex(2); |
|
488 |
|
|
489 |
SailRepositoryConnection connection = null; |
|
490 |
try { |
|
491 |
|
|
492 |
Profiler profiler = Profiler.newCpuProfiler(true); |
|
493 |
|
|
494 |
connection = queryClient.connection(); |
|
495 |
TupleQuery tquery = connection.prepareTupleQuery(QueryLanguage.SPARQL, sparql.toString()); |
|
496 |
TupleQueryResult tqresult = tquery.evaluate(); |
|
497 |
queryClient.showResults(tqresult); |
|
498 |
|
|
499 |
profiler.end(System.err); |
|
500 |
|
|
501 |
} catch (MalformedQueryException | RepositoryException | QueryEvaluationException e1) { |
|
502 |
// TODO Auto-generated catch block |
|
503 |
e1.printStackTrace(); |
|
504 |
} catch (Exception e1) { |
|
505 |
// yourkit |
|
506 |
e1.printStackTrace(); |
|
507 |
} finally { |
|
508 |
try { |
|
509 |
connection.close(); |
|
510 |
} catch (RepositoryException e1) { |
|
511 |
// IGNORE // |
|
512 |
} |
|
513 |
connection = null; |
|
514 |
} |
|
515 |
|
|
516 |
} |
|
517 |
GremlinPipeline<Graph, Vertex> pipe = null; |
|
518 |
|
|
519 |
if(Neo4jINDEX) { |
|
520 |
|
|
521 |
Profiler profiler = Profiler.newCpuProfiler(false); |
|
522 |
|
|
523 |
logger.debug("Neo4jINDEX"); |
|
524 |
Collection<Vertex> resultSet = new ArrayList<Vertex>(); |
|
525 |
|
|
526 |
// Theoretically it should be sufficient to use a Gremlin Pipe to find |
|
527 |
// the species nodes but gremlin is obviously not using the Neo4j index |
|
528 |
// to it is far too slow ... |
|
529 |
// |
|
530 |
//GremlinPipeline<Graph, Object> pipe2 = new GremlinPipeline<Graph, Object>( |
|
531 |
// graph.getVertices("http://eunis.eea.europa.eu/rdf/species-schema.rdf#binomialName", queryString) |
|
532 |
// ); |
|
533 |
|
|
534 |
// ... therefore the Neo4j index is used directly |
|
535 |
Neo4jGraph graph = (Neo4jGraph)queryClient.graph(); |
|
536 |
Index<Neo4jVertex> vertexAutoIndex = graph.getIndex("node_auto_index", Neo4jVertex.class); |
|
537 |
CloseableIterable<Neo4jVertex> nodes = vertexAutoIndex.query("value", "\"" + queryString + "\""); |
|
538 |
|
|
539 |
pipe = new GremlinPipeline<Graph, Vertex>(nodes); |
|
540 |
|
|
541 |
List<Vertex> vertices = new ArrayList<Vertex>(); |
|
542 |
pipe.in("http://eunis.eea.europa.eu/rdf/species-schema.rdf#binomialName").fill(vertices); |
|
543 |
|
|
544 |
|
|
545 |
for(Vertex v : vertices) { |
|
546 |
logger.debug(" " + v.toString()); |
|
547 |
} |
|
548 |
|
|
549 |
// for (Iterator<Neo4jVertex> it = nodes.iterator(); it.hasNext();) { |
|
550 |
// Vertex v = it.next(); |
|
551 |
// logger.debug(" " + v.toString()); |
|
552 |
// vertices.add(graph.getVertex(v.getId())); |
|
553 |
// } |
|
554 |
|
|
555 |
|
|
556 |
nodes.close(); |
|
557 |
|
|
558 |
profiler.end(System.err); |
|
559 |
updateQueriesWithResponse(vertices, checklistInfo, query); |
|
560 |
} |
|
411 | 561 |
|
412 |
logger.debug("\n" + queryString.toString()); |
|
413 |
/* FIXME |
|
414 |
Model model = queryClient.describe(queryString.toString()); |
|
415 |
updateQueriesWithResponse(model, checklistInfo, query); */ |
|
416 | 562 |
} |
417 | 563 |
} |
418 | 564 |
|
... | ... | |
426 | 572 |
|
427 | 573 |
@Override |
428 | 574 |
public void resolveVernacularNamesExact(TnrMsg tnrMsg) throws DRFChecklistException { |
575 |
/* |
|
429 | 576 |
List<Query> queryList = tnrMsg.getQuery(); |
430 | 577 |
|
431 | 578 |
// selecting one request as representative, only |
... | ... | |
455 | 602 |
); |
456 | 603 |
|
457 | 604 |
logger.debug("\n" + queryString.toString()); |
605 |
*/ |
|
458 | 606 |
/* FIXME |
459 | 607 |
Model model = queryClient.describe(queryString.toString()); |
460 | 608 |
updateQueriesWithResponse(model, checklistInfo, query); */ |
461 |
} |
|
462 |
|
|
609 |
// } |
|
463 | 610 |
} |
464 | 611 |
|
465 | 612 |
@Override |
... | ... | |
469 | 616 |
|
470 | 617 |
@Override |
471 | 618 |
public void findByIdentifier(TnrMsg tnrMsg) throws DRFChecklistException { |
472 |
for (ServiceProviderInfo checklistInfo : getServiceProviderInfo().getSubChecklists()) { |
|
619 |
/*for (ServiceProviderInfo checklistInfo : getServiceProviderInfo().getSubChecklists()) {
|
|
473 | 620 |
|
474 | 621 |
Query query = singleQueryFrom(tnrMsg); |
475 | 622 |
Resource taxonR = queryClient.getFromUri(query.getRequest().getQueryString()); |
... | ... | |
477 | 624 |
Response response = tnrResponseFromResource(taxonR.getModel(), taxonR, query.getRequest()); |
478 | 625 |
query.getResponse().add(response); |
479 | 626 |
} |
627 |
*/ |
|
480 | 628 |
} |
481 | 629 |
|
482 |
private void updateQueriesWithResponse(Model model, ServiceProviderInfo ci, Query query) |
|
483 |
throws DRFChecklistException { |
|
630 |
private void updateQueriesWithResponse(List<Vertex> nodes, ServiceProviderInfo ci, Query query){ |
|
484 | 631 |
|
485 |
if (model == null) {
|
|
632 |
if (nodes == null) {
|
|
486 | 633 |
return; |
487 | 634 |
} |
488 | 635 |
|
489 |
ResIterator subjectIt = model.listSubjects(); |
|
490 |
|
|
491 |
while (subjectIt.hasNext()) { |
|
492 |
Resource subject = subjectIt.next(); |
|
493 |
Resource taxonR; |
|
494 |
StmtIterator exactMatches = subject.listProperties(subject.getModel().getProperty(RdfSchema.SKOS_CORE.schemaUri, "exactMatch")); |
|
495 |
if(exactMatches.hasNext()) { |
|
496 |
// need to follow the exactMatch uri in this case |
|
497 |
taxonR = queryClient.getFromUri(exactMatches.next().getResource().getURI()); |
|
498 |
} else { |
|
499 |
// the subject is already a species |
|
500 |
taxonR = subject; |
|
636 |
logger.debug("matching nodes:"); |
|
637 |
for (Vertex v : nodes) { |
|
638 |
logger.debug(" " + v.toString()); |
|
639 |
printPropertyKeys(v, System.err); |
|
640 |
if(v.getProperty("kind").equals("url")) { |
|
641 |
logger.error("vertex of type 'url' expected, but was " + v.getProperty("type").equals("url")); |
|
642 |
continue; |
|
501 | 643 |
} |
502 |
|
|
503 |
Response tnrResponse = tnrResponseFromResource(model, taxonR, query.getRequest()); |
|
644 |
Response tnrResponse = tnrResponseFromResource(v, query.getRequest()); |
|
504 | 645 |
if(tnrResponse != null) { |
505 | 646 |
query.getResponse().add(tnrResponse); |
506 | 647 |
} |
... | ... | |
513 | 654 |
* @param request |
514 | 655 |
* @return |
515 | 656 |
*/ |
516 |
private Response tnrResponseFromResource(Model model, Resource taxonR, Request request) { |
|
657 |
@SuppressWarnings("unused") |
|
658 |
private Response tnrResponseFromResource(Vertex taxonV, Request request) { |
|
517 | 659 |
|
518 | 660 |
Response tnrResponse = TnrMsgUtils.tnrResponseFor(getServiceProviderInfo()); |
519 | 661 |
|
520 | 662 |
SearchMode searchMode = SearchMode.valueOf(request.getSearchMode()); |
521 | 663 |
|
522 |
// Check for type to ignore the triple pointing from synonyms to accepted taxonUris |
|
523 |
// only complete descriptions of taxa and synonym are relevant. |
|
524 |
boolean isCompleteResource = taxonR.hasProperty(taxonR.getModel().getProperty(RdfSchema.RDF.schemaUri, "type")); |
|
525 |
if(!isCompleteResource) { |
|
526 |
return null; |
|
527 |
} |
|
664 |
GremlinPipeline<Graph, Vertex> pipe = new GremlinPipeline<Graph, Vertex>(taxonV); |
|
665 |
|
|
666 |
String validName = queryClient.relatedVertexValue(taxonV, RdfSchema.EUNIS_SPECIES, "validName"); |
|
528 | 667 |
|
529 |
String validName = queryClient.objectAsString(taxonR, RdfSchema.EUNIS_SPECIES, "validName"); |
|
530 |
boolean isAccepted = validName != null && validName.equals("true^^http://www.w3.org/2001/XMLSchema#boolean"); |
|
668 |
boolean isAccepted = validName != null && validName.equals("true"); |
|
531 | 669 |
boolean skipThis = false; |
532 | 670 |
|
533 |
logger.debug("processing " + (isAccepted ? "accepted taxon" : "synonym or other") + " " + taxonR.getURI());
|
|
671 |
logger.debug("processing " + (isAccepted ? "accepted taxon" : "synonym or other") + " " + taxonV.getId());
|
|
534 | 672 |
|
535 | 673 |
// case when accepted name |
536 | 674 |
if(isAccepted) { |
537 |
Taxon taxon = createTaxon(model, taxonR);
|
|
675 |
Taxon taxon = createTaxon(taxonV);
|
|
538 | 676 |
tnrResponse.setTaxon(taxon); |
539 | 677 |
tnrResponse.setMatchingNameType(NameType.TAXON); |
540 | 678 |
String matchingName = taxon.getTaxonName().getCanonicalName(); |
541 | 679 |
tnrResponse.setMatchingNameString(matchingName); |
542 | 680 |
|
543 |
} else { |
|
681 |
} |
|
682 |
else { |
|
544 | 683 |
// case when synonym |
545 |
Resource synonymR = taxonR; |
|
546 |
URI taxonUri = queryClient.objectAsURI(taxonR, RdfSchema.EUNIS_SPECIES, "eunisPrimaryName"); |
|
547 |
if(taxonUri == null) { |
|
548 |
logger.error("no taxon uri found"); |
|
684 |
Vertex synonymV = taxonV; |
|
685 |
taxonV = null; |
|
686 |
try { |
|
687 |
taxonV = synonymV.getEdges(Direction.OUT, RdfSchema.EUNIS_SPECIES.propertyURI("eunisPrimaryName")).iterator().next().getVertex(Direction.IN); |
|
688 |
} catch(Exception e) { |
|
689 |
logger.error("No accepted taxon found for " + synonymV.toString() + " (" + synonymV.getProperty(GraphSail.VALUE) + ")"); |
|
549 | 690 |
} |
550 | 691 |
|
551 |
taxonR = queryClient.getFromUri(taxonUri); |
|
552 |
if(taxonR != null) { |
|
553 |
Taxon taxon = createTaxon(model, taxonR); |
|
692 |
if(taxonV != null) { |
|
693 |
Taxon taxon = createTaxon(taxonV); |
|
554 | 694 |
tnrResponse.setTaxon(taxon); |
555 | 695 |
} else { |
556 |
logger.error("No accepted taxon found for " + synonymR.getURI()); |
|
557 | 696 |
} |
558 | 697 |
tnrResponse.setMatchingNameType(NameType.SYNONYM); |
559 |
String matchingName = queryClient.objectAsString(synonymR, RdfSchema.EUNIS_SPECIES, "binomialName");
|
|
698 |
String matchingName = queryClient.relatedVertexValue(synonymV, RdfSchema.EUNIS_SPECIES, "binomialName");
|
|
560 | 699 |
tnrResponse.setMatchingNameString(matchingName); |
561 | 700 |
} |
562 | 701 |
|
563 | 702 |
if(!skipThis && request.isAddSynonymy()) { |
564 |
createSynonyms(taxonR, tnrResponse);
|
|
703 |
createSynonyms(taxonV, tnrResponse);
|
|
565 | 704 |
} |
566 |
logger.debug("processing " + (isAccepted ? "accepted taxon" : "synonym or other") + " " + taxonR.getURI() + " DONE"); |
|
567 | 705 |
|
706 |
logger.debug("processing " + (isAccepted ? "accepted taxon" : "synonym or other") + " " + taxonV.getId() + " DONE"); |
|
568 | 707 |
return tnrResponse; |
569 | 708 |
} |
570 | 709 |
|
710 |
/** |
|
711 |
* @param vertex |
|
712 |
*/ |
|
713 |
private void printEdges(Neo4jVertex vertex) { |
|
714 |
Iterable<Relationship> rels = vertex.getRawVertex().getRelationships(); |
|
715 |
Iterator<Relationship> iterator = rels.iterator(); |
|
716 |
if(iterator.hasNext()) { |
|
717 |
Relationship rel = iterator.next(); |
|
718 |
System.err.println(rel.toString() + ": " + rel.getStartNode().toString() + "-[" + rel.getType() + "]-" + rel.getEndNode().toString()); |
|
719 |
} |
|
720 |
} |
|
721 |
|
|
722 |
private void printPropertyKeys(Vertex v, PrintStream ps) { |
|
723 |
StringBuilder out = new StringBuilder(); |
|
724 |
out.append(v.toString()); |
|
725 |
for(String key : v.getPropertyKeys()) { |
|
726 |
out.append(key).append(": ").append(v.getProperty(key)).append(" "); |
|
727 |
} |
|
728 |
ps.println(out.toString()); |
|
729 |
} |
|
730 |
|
|
571 | 731 |
@Override |
572 | 732 |
public EnumSet<SearchMode> getSearchModes() { |
573 | 733 |
return SEARCH_MODES; |
src/main/java/org/bgbm/biovel/drf/query/TinkerPopClient.java | ||
---|---|---|
17 | 17 |
import org.apache.jena.rdf.model.Model; |
18 | 18 |
import org.apache.jena.rdf.model.Property; |
19 | 19 |
import org.apache.jena.rdf.model.RDFNode; |
20 |
import org.apache.jena.rdf.model.ResIterator; |
|
21 | 20 |
import org.apache.jena.rdf.model.Resource; |
22 | 21 |
import org.apache.jena.rdf.model.StmtIterator; |
23 |
import org.apache.jena.riot.RDFDataMgr; |
|
24 |
import org.apache.jena.riot.RDFFormat; |
|
25 | 22 |
import org.bgbm.biovel.drf.checklist.DRFChecklistException; |
26 | 23 |
import org.bgbm.biovel.drf.checklist.EEA_BDC_Client.RdfSchema; |
27 | 24 |
import org.bgbm.biovel.drf.store.Neo4jStore; |
... | ... | |
32 | 29 |
import org.openrdf.query.TupleQuery; |
33 | 30 |
import org.openrdf.query.TupleQueryResult; |
34 | 31 |
import org.openrdf.repository.RepositoryException; |
32 |
import org.openrdf.repository.sail.SailRepositoryConnection; |
|
35 | 33 |
import org.slf4j.Logger; |
36 | 34 |
import org.slf4j.LoggerFactory; |
37 | 35 |
|
36 |
import com.tinkerpop.blueprints.Direction; |
|
37 |
import com.tinkerpop.blueprints.Graph; |
|
38 |
import com.tinkerpop.blueprints.Vertex; |
|
39 |
import com.tinkerpop.blueprints.oupls.sail.GraphSail; |
|
40 |
import com.tinkerpop.gremlin.java.GremlinPipeline; |
|
41 |
import com.tinkerpop.pipes.PipeFunction; |
|
42 |
import com.tinkerpop.pipes.util.FastNoSuchElementException; |
|
43 |
|
|
38 | 44 |
/** |
39 | 45 |
* |
40 | 46 |
* Developer links: |
41 | 47 |
* <ul> |
42 | 48 |
* <li>https://github.com/tinkerpop/blueprints/wiki/Sail-Ouplementation</li> |
43 | 49 |
* <li>https://github.com/tinkerpop/gremlin/wiki</li> |
44 |
* <li>http://rdf4j.org/sesame/2.7/docs/users.docbook?view#section-repository-api3</li> |
|
50 |
* <li> |
|
51 |
* http://rdf4j.org/sesame/2.7/docs/users.docbook?view#section-repository-api3</li> |
|
45 | 52 |
* <li>https://github.com/tinkerpop/gremlin/wiki/SPARQL-vs.-Gremlin</li> |
46 | 53 |
* <li>https://github.com/tinkerpop/gremlin/wiki/Using-Gremlin-through-Java</li> |
54 |
* <li> |
|
55 |
* http://markorodriguez.com/2011/06/15/graph-pattern-matching-with-gremlin-1-1/ |
|
56 |
* </li> |
|
57 |
* <li>https://github.com/datablend/neo4j-sail-test</li> |
|
47 | 58 |
* </ul> |
59 |
* |
|
48 | 60 |
* @author a.kohlbecker |
49 | 61 |
* @date Sep 30, 2015 |
50 | 62 |
* |
51 | 63 |
*/ |
52 | 64 |
public class TinkerPopClient implements IQueryClient { |
53 | 65 |
|
66 |
public static final String KIND = GraphSail.KIND; |
|
67 |
|
|
68 |
public static final String VALUE = GraphSail.VALUE; |
|
69 |
|
|
70 |
public static final Object URI = GraphSail.URI; |
|
71 |
|
|
54 | 72 |
protected Logger logger = LoggerFactory.getLogger(TinkerPopClient.class); |
55 | 73 |
|
56 | 74 |
private final String baseUri = null; |
... | ... | |
58 | 76 |
private Neo4jStore tripleStore = null; |
59 | 77 |
|
60 | 78 |
/** |
61 |
* A model for caching
|
|
79 |
* @param tripleStore
|
|
62 | 80 |
*/ |
63 |
private final Model cache = null; |
|
81 |
public TinkerPopClient(Neo4jStore tripleStore) { |
|
82 |
this.tripleStore = tripleStore; |
|
83 |
} |
|
64 | 84 |
|
85 |
public GremlinPipeline<Graph, Graph> newPipe() { |
|
86 |
GremlinPipeline<Graph, Object> pipe = new GremlinPipeline<Graph, Object>(); |
|
87 |
return pipe.start(graph()); |
|
88 |
} |
|
89 |
|
|
90 |
public Graph graph() { |
|
91 |
return tripleStore.graph(); |
|
92 |
} |
|
65 | 93 |
|
66 | 94 |
/** |
67 |
* @param tripleStore |
|
95 |
* @param sparql |
|
96 |
* @return |
|
97 |
* @throws MalformedQueryException |
|
98 |
* @throws RepositoryException |
|
99 |
* @throws QueryEvaluationException |
|
100 |
* |
|
101 |
* @deprecated directly use connection() and do not forget to close is after |
|
102 |
* doing the query. See |
|
103 |
* {@link org.openrdf.repository.sail.SailRepository#getConnection()} |
|
68 | 104 |
*/ |
69 |
public TinkerPopClient(Neo4jStore tripleStore) { |
|
70 |
this.tripleStore = tripleStore; |
|
105 |
@Deprecated |
|
106 |
public TupleQueryResult execute(String sparql) throws MalformedQueryException, RepositoryException, |
|
107 |
QueryEvaluationException { |
|
108 |
TupleQuery query = connection().prepareTupleQuery(QueryLanguage.SPARQL, sparql); |
|
109 |
TupleQueryResult result = query.evaluate(); |
|
110 |
return result; |
|
111 |
} |
|
112 |
|
|
113 |
/** |
|
114 |
* See {@link org.openrdf.repository.sail.SailRepository#getConnection()} |
|
115 |
* |
|
116 |
* @return |
|
117 |
* @throws RepositoryException |
|
118 |
*/ |
|
119 |
public SailRepositoryConnection connection() throws RepositoryException { |
|
120 |
SailRepositoryConnection connection = tripleStore.getSailRepo().getConnection(); |
|
121 |
return connection; |
|
122 |
} |
|
123 |
|
|
124 |
public PipeFunction<Vertex, Boolean> createRegexMatchFilter(final String regex) { |
|
125 |
return new PipeFunction<Vertex, Boolean>() { |
|
126 |
|
|
127 |
@Override |
|
128 |
public Boolean compute(Vertex v) { |
|
129 |
return v.toString().matches(regex); |
|
130 |
} |
|
131 |
}; |
|
132 |
} |
|
133 |
|
|
134 |
public PipeFunction<Vertex, Boolean> createEqualsFilter(final String string) { |
|
135 |
return new PipeFunction<Vertex, Boolean>() { |
|
136 |
|
|
137 |
@Override |
|
138 |
public Boolean compute(Vertex v) { |
|
139 |
return v.toString().equals(string); |
|
140 |
} |
|
141 |
}; |
|
142 |
} |
|
143 |
|
|
144 |
public PipeFunction<Vertex, Boolean> createStarttWithFilter(final String string) { |
|
145 |
return new PipeFunction<Vertex, Boolean>() { |
|
146 |
|
|
147 |
@Override |
|
148 |
public Boolean compute(Vertex v) { |
|
149 |
return v.toString().startsWith(string); |
|
150 |
} |
|
151 |
}; |
|
71 | 152 |
} |
72 | 153 |
|
73 | 154 |
public Model describe(String queryString) throws DRFChecklistException, QueryEvaluationException { |
... | ... | |
79 | 160 |
TupleQueryResult result = qe.evaluate(); |
80 | 161 |
System.err.println(result.toString()); |
81 | 162 |
|
82 |
if(result != null && logger.isDebugEnabled()) { |
|
163 |
if (result != null && logger.isDebugEnabled()) {
|
|
83 | 164 |
StringBuilder msg = new StringBuilder(); |
84 | 165 |
msg.append("subjects in response:\n"); |
85 | 166 |
int i = 1; |
86 | 167 |
try { |
87 |
for(; result.hasNext(); ++i) { |
|
168 |
for (; result.hasNext(); ++i) {
|
|
88 | 169 |
BindingSet res = result.next(); |
89 | 170 |
msg.append(" " + i + ": " + res.toString() + "\n"); |
90 | 171 |
} |
... | ... | |
95 | 176 |
logger.debug(msg.toString()); |
96 | 177 |
} |
97 | 178 |
|
98 |
return null; //FIXME result; |
|
179 |
return null; // FIXME result;
|
|
99 | 180 |
} |
100 | 181 |
|
101 | 182 |
/** |
... | ... | |
104 | 185 |
*/ |
105 | 186 |
private TupleQuery executionFor(String queryString) { |
106 | 187 |
|
107 |
|
|
108 |
if(baseUri != null) {
|
|
109 |
// see https://github.com/tinkerpop/blueprints/wiki/Sail-Implementation
|
|
188 |
if (baseUri != null) { |
|
189 |
// see
|
|
190 |
// https://github.com/tinkerpop/blueprints/wiki/Sail-Implementation |
|
110 | 191 |
// FIXME |
111 | 192 |
throw new RuntimeException("Mode unsupported"); |
112 | 193 |
// Graph graph = new SparqlRepositorySailGraph(baseUri); |
113 |
//return QueryExecutionFactory.sparqlService(baseUri, query); |
|
194 |
// return QueryExecutionFactory.sparqlService(baseUri, query);
|
|
114 | 195 |
} |
115 |
if(tripleStore != null) { |
|
196 |
if (tripleStore != null) {
|
|
116 | 197 |
// local TDB Store |
117 | 198 |
try { |
118 |
return tripleStore.connection().prepareTupleQuery(QueryLanguage.SPARQL, queryString); |
|
199 |
SailRepositoryConnection connection = connection(); |
|
200 |
return connection.prepareTupleQuery(QueryLanguage.SPARQL, queryString); |
|
119 | 201 |
} catch (MalformedQueryException | RepositoryException e) { |
120 | 202 |
// TODO Auto-generated catch block |
121 | 203 |
logger.error("Error while perparing query", e); |
... | ... | |
143 | 225 |
propertyIt = _subject.listProperties(property); |
144 | 226 |
|
145 | 227 |
boolean propertyInGraph = propertyIt.hasNext(); |
146 |
if(!propertyInGraph ) {
|
|
228 |
if (!propertyInGraph) {
|
|
147 | 229 |
_subject = getFromUri(subject.getURI()); |
148 | 230 |
propertyIt = _subject.listProperties(property); |
149 | 231 |
} |
150 | 232 |
|
151 | 233 |
node = propertyIt.next().getObject(); |
152 | 234 |
} catch (NoSuchElementException e) { |
153 |
if(logger.isTraceEnabled()) { |
|
154 |
logger.debug(_subject.getURI() + " " + nameSpace + ":" + localName + " not found in current graph");
|
|
235 |
if (logger.isTraceEnabled()) {
|
|
236 |
logger.debug(_subject.getURI() + " " + nameSpace + ":" + localName + " not found in current graph"); |
|
155 | 237 |
printProperties(_subject); |
156 | 238 |
} |
157 | 239 |
} |
... | ... | |
175 | 257 |
propertyIt = _subject.listProperties(property); |
176 | 258 |
|
177 | 259 |
boolean propertyInGraph = propertyIt.hasNext(); |
178 |
if(!propertyInGraph ) {
|
|
260 |
if (!propertyInGraph) {
|
|
179 | 261 |
_subject = getFromUri(subject.getURI()); |
180 | 262 |
propertyIt = _subject.listProperties(property); |
181 | 263 |
} |
182 | 264 |
|
183 | 265 |
} catch (NoSuchElementException e) { |
184 |
if(logger.isTraceEnabled()) { |
|
185 |
logger.debug(_subject.getURI() + " " + nameSpace + ":" + localName + " not found in current graph");
|
|
266 |
if (logger.isTraceEnabled()) {
|
|
267 |
logger.debug(_subject.getURI() + " " + nameSpace + ":" + localName + " not found in current graph"); |
|
186 | 268 |
printProperties(_subject); |
187 | 269 |
} |
188 | 270 |
} |
189 | 271 |
return propertyIt; |
190 | 272 |
} |
191 | 273 |
|
192 |
public List<RDFNode> listObjects(Resource subject, RdfSchema nameSpace, String localName) {
|
|
274 |
public List<RDFNode> listObjects(Resource subject, RdfSchema nameSpace, String localName) { |
|
193 | 275 |
|
194 | 276 |
List<RDFNode> list = new ArrayList<RDFNode>(); |
195 | 277 |
StmtIterator it = listProperties(subject, nameSpace, localName); |
... | ... | |
205 | 287 |
* @param localName |
206 | 288 |
* @return |
207 | 289 |
*/ |
208 |
public String objectAsString(Resource subject, RdfSchema nameSpace, String localName) {
|
|
290 |
public String relatedVertexValue(GremlinPipeline<Graph, Vertex> pipe, RdfSchema nameSpace, String localName) {
|
|
209 | 291 |
String txt = null; |
210 |
RDFNode node = asSingleObject(subject, nameSpace, localName); |
|
211 |
if(node != null) { |
|
212 |
txt = node.toString(); |
|
292 |
String edgeLabel = nameSpace.propertyURI(localName); |
|
293 |
try { |
|
294 |
txt = pipe.outE(1, edgeLabel).inV().next().getProperty("value"); |
|
295 |
} catch (FastNoSuchElementException e) { |
|
296 |
try { |
|
297 |
txt = pipe.inE(1, edgeLabel).inV().next().getProperty("value"); |
|
298 |
} catch (FastNoSuchElementException e1) { |
|
299 |
logger.warn("edge with '" + edgeLabel + "' not found"); |
|
300 |
if (logger.isDebugEnabled()) { |
|
301 |
logger.debug("Vertices in pipe:"); |
|
302 |
StringBuffer out = new StringBuffer(); |
|
303 |
for (Vertex v : pipe.toList()) { |
|
304 |
logger.debug(" " + v); |
|
305 |
} |
|
306 |
} |
|
307 |
} |
|
213 | 308 |
} |
309 |
|
|
214 | 310 |
return txt; |
215 | 311 |
} |
216 | 312 |
|
313 |
public String relatedVertexValue(Vertex v, RdfSchema nameSpace, String localName) { |
|
314 |
String txt = null; |
|
315 |
String edgeLabel = nameSpace.propertyURI(localName); |
|
316 |
try { |
|
317 |
txt = v.getEdges(Direction.OUT, edgeLabel).iterator().next().getVertex(Direction.IN) |
|
318 |
.getProperty(GraphSail.VALUE); |
|
319 |
} catch (NoSuchElementException e) { |
|
320 |
try { |
|
321 |
txt = v.getEdges(Direction.IN, edgeLabel).iterator().next().getVertex(Direction.OUT) |
|
322 |
.getProperty(GraphSail.VALUE); |
|
323 |
} catch (NoSuchElementException e1) { |
|
324 |
logger.warn("edge with '" + edgeLabel + "' not found for " + v); |
|
325 |
|
|
326 |
} |
|
327 |
} |
|
328 |
|
|
329 |
return txt; |
|
330 |
} |
|
331 |
|
|
332 |
// |
|
333 |
|
|
217 | 334 |
/** |
218 | 335 |
* @param subject |
219 | 336 |
* @param nameSpace |
... | ... | |
223 | 340 |
public Resource objectAsResource(Resource subject, RdfSchema nameSpace, String localName) { |
224 | 341 |
Resource resource = null; |
225 | 342 |
RDFNode node = asSingleObject(subject, nameSpace, localName); |
226 |
if(node != null) { |
|
343 |
if (node != null) {
|
|
227 | 344 |
node.isResource(); |
228 |
resource = node.asResource();
|
|
345 |
resource = node.asResource(); |
|
229 | 346 |
} |
230 | 347 |
return resource; |
231 | 348 |
} |
... | ... | |
235 | 352 |
* @param nameSpace |
236 | 353 |
* @param localName |
237 | 354 |
* @return |
355 |
* @deprecated unused |
|
238 | 356 |
*/ |
239 |
public URI objectAsURI(Resource subject, RdfSchema nameSpace, String localName) { |
|
357 |
@Deprecated |
|
358 |
public URI relatedVertexURI(GremlinPipeline<Graph, Vertex> pipe, RdfSchema nameSpace, String localName) { |
|
240 | 359 |
URI uri = null; |
241 |
RDFNode node = asSingleObject(subject, nameSpace, localName); |
|
242 |
if(node != null) { |
|
243 |
node.isURIResource(); |
|
244 |
try { |
|
245 |
uri = new URI(node.asResource().getURI()); |
|
246 |
} catch (URISyntaxException e) { |
|
247 |
// this should actually never happen |
|
248 |
throw new RuntimeException(e); |
|
360 |
String edgeLabel = nameSpace.propertyURI(localName); |
|
361 |
try { |
|
362 |
Vertex v = pipe.outE(1, edgeLabel).inV().next(); |
|
363 |
if (v.getProperty(GraphSail.KIND).equals(GraphSail.URI)) { |
|
364 |
uri = (URI) v.getProperty(GraphSail.VALUE); |
|
365 |
} else { |
|
366 |
logger.warn("target vertex of '" + edgeLabel + "' is not an URI"); |
|
249 | 367 |
} |
368 |
} catch (FastNoSuchElementException e) { |
|
369 |
logger.warn("edge with '" + edgeLabel + "' not found"); |
|
250 | 370 |
} |
371 |
|
|
251 | 372 |
return uri; |
252 | 373 |
} |
253 | 374 |
|
254 |
public List<Resource> listResources(Model model, Property filterProperty, String filterValue, Resource exclude){ |
|
255 |
|
|
256 |
List<Resource> resultList = new ArrayList<Resource>(); |
|
257 |
ResIterator resItr = model.listSubjects(); |
|
258 |
while(resItr.hasNext()) { |
|
259 |
Resource resource = resItr.next(); |
|
260 |
|
|
261 |
if(exclude != null && resource.getURI().equals(exclude.getURI())) { |
|
262 |
continue; |
|
263 |
} |
|
264 |
|
|
265 |
boolean hasProperty = true; |
|
266 |
if(filterProperty != null) { |
|
267 |
if(filterValue != null) { |
|
268 |
hasProperty = resource.hasProperty(filterProperty, filterValue); |
|
269 |
} else { |
|
270 |
hasProperty = resource.hasProperty(filterProperty); |
|
271 |
} |
|
272 |
} |
|
273 |
|
|
274 |
if(filterProperty == null || hasProperty) { |
|
275 |
resultList.add(resource); |
|
276 |
logger.debug("adding " + resource.getURI()); |
|
375 |
/** |
|
376 |
* @param subject |
|
377 |
* @param nameSpace |
|
378 |
* @param localName |
|
379 |
* @return |
|
380 |
* @deprecated possibly broken |
|
381 |
*/ |
|
382 |
@Deprecated |
|
383 |
public URI relatedVertexURI(Vertex v, RdfSchema nameSpace, String localName) { |
|
384 |
URI uri = null; |
|
385 |
String edgeLabel = nameSpace.propertyURI(localName); |
|
386 |
try { |
|
387 |
if (v.getProperty(GraphSail.KIND).equals(GraphSail.URI)) { |
|
388 |
uri = new URI(v.getProperty(GraphSail.VALUE).toString()); |
|
277 | 389 |
} else { |
278 |
logger.debug("skipping " + resource.getURI());
|
|
390 |
logger.warn("target vertex of '" + edgeLabel + "' is not an URI");
|
|
279 | 391 |
} |
392 |
} catch (NoSuchElementException e) { |
|
393 |
logger.warn("edge with '" + edgeLabel + "' not found"); |
|
394 |
} catch (URISyntaxException e) { |
|
395 |
logger.error("Invalid URI id in " + v, e); |
|
280 | 396 |
} |
281 |
return resultList; |
|
397 |
|
|
398 |
return uri; |
|
282 | 399 |
} |
283 | 400 |
|
284 | 401 |
/** |
285 | 402 |
* @param subject |
286 | 403 |
*/ |
287 | 404 |
private void printProperties(Resource subject) { |
288 |
for( StmtIterator it = subject.listProperties(); it.hasNext(); ) {
|
|
405 |
for (StmtIterator it = subject.listProperties(); it.hasNext();) {
|
|
289 | 406 |
System.err.println(it.next().toString()); |
290 | 407 |
} |
291 | 408 |
} |
... | ... | |
301 | 418 |
|
302 | 419 |
public Resource getFromUri(String uri) { |
303 | 420 |
|
304 |
Model model = null; |
|
305 |
if(tripleStore != null) { |
|
306 |
/* FIXME |
|
307 |
Dataset dataset = tripleStore.getDataset(); |
|
308 |
dataset.begin(ReadWrite.READ) ; |
|
309 |
model = dataset.getDefaultModel(); |
|
310 |
dataset.end(); |
|
311 |
*/ |
|
312 |
} else { |
|
313 |
model = cache; |
|
314 |
// FIXME the same uri resource is loaded from remote multiple times |
|
315 |
// create an in memory model as cache for the models loaded |
|
316 |
// in the getFromUri |
|
317 |
// so that all resources loaded are put into that model |
|
318 |
// clean up the cache when it reaches a specific size |
|
319 |
logger.debug("loading remote UriResource " + uri); |
|
320 |
model.read(uri); |
|
321 |
} |
|
322 |
if(logger.isDebugEnabled()) { |
|
323 |
// see https://jena.apache.org/documentation/io/rdf-output.html#examples |
|
324 |
RDFDataMgr.write(System.err, model, RDFFormat.TURTLE_PRETTY); |
|
325 |
} |
|
326 |
return model.getResource(uri); |
|
421 |
// not needed |
|
422 |
return null; |
|
327 | 423 |
|
328 | 424 |
} |
329 | 425 |
|
330 |
|
|
331 |
|
|
426 |
public void showResults(TupleQueryResult result) throws QueryEvaluationException { |
|
427 |
int i = 0; |
|
428 |
while (result.hasNext()) { |
|
429 |
System.err.println("."); |
|
430 |
BindingSet bindingSet = result.next(); |
|
431 |
System.err.println("+"); |
|
432 |
for (String colName : result.getBindingNames()) { |
|
433 |
System.err.println(colName + ":" + bindingSet.getValue(colName)); |
|
434 |
} |
|
435 |
System.err.println("|"); |
|
436 |
i++; |
|
437 |
} |
|
438 |
System.err.println("We found " + i + " results"); |
|
439 |
System.err.flush(); |
|
440 |
} |
|
332 | 441 |
} |
src/main/java/org/bgbm/biovel/drf/store/Neo4jStore.java | ||
---|---|---|
11 | 11 |
|
12 | 12 |
import java.io.File; |
13 | 13 |
|
14 |
import org.openrdf.repository.base.RepositoryConnectionBase; |
|
15 | 14 |
import org.openrdf.repository.sail.SailRepository; |
16 | 15 |
import org.openrdf.sail.Sail; |
17 | 16 |
import org.openrdf.sail.SailException; |
... | ... | |
29 | 28 |
*/ |
30 | 29 |
public class Neo4jStore extends Store{ |
31 | 30 |
|
31 |
private Neo4jGraph graph; |
|
32 |
private Sail sail; |
|
33 |
private SailRepository sailRepo; |
|
32 | 34 |
|
33 | 35 |
/** |
34 | 36 |
* @throws Exception |
... | ... | |
37 | 39 |
super(); |
38 | 40 |
} |
39 | 41 |
|
40 |
private RepositoryConnectionBase connection; |
|
41 |
private Neo4jGraph graph; |
|
42 |
private Sail sail = null; |
|
43 |
|
|
44 | 42 |
|
45 | 43 |
/** |
46 | 44 |
* {@inheritDoc} |
... | ... | |
50 | 48 |
protected void initStoreEngine() throws Exception { |
51 | 49 |
|
52 | 50 |
graph = new Neo4jGraph(storeLocation.toString()); |
53 |
sail = new GraphSail(graph); |
|
51 |
sail = new GraphSail<Neo4jGraph>(graph);
|
|
54 | 52 |
sail.initialize(); |
55 |
logger.info("Using Neo4jGraph store at " + storeLocation.toString()); |
|
56 |
logger.info("Neo4jGraph has " + sizeInfo()); |
|
53 |
sailRepo = new SailRepository(sail); |
|
54 |
|
|
55 |
// logger.info("Using Neo4jGraph store at " + storeLocation.toString()); |
|
56 |
// logger.info("Neo4jGraph has " + sizeInfo()); |
|
57 | 57 |
} |
58 | 58 |
|
59 |
/** |
|
60 |
* {@inheritDoc} |
|
61 |
*/ |
|
62 |
@Override |
|
63 |
protected void stopStoreEngine() throws Exception { |
|
64 |
sailRepo.shutDown(); |
|
65 |
sail.shutDown(); // should be none by the above command already |
|
66 |
} |
|
59 | 67 |
|
60 | 68 |
/** |
61 | 69 |
* @throws Exception |
... | ... | |
65 | 73 |
protected void load(File rdfFile) throws Exception { |
66 | 74 |
|
67 | 75 |
SailLoader loader = new SailLoader(sail); |
68 |
// loader.setBufferSize(100000); |
|
76 |
// loader.setBufferSize(100000); // TODO optimize?
|
|
69 | 77 |
logger.info("loading RDF/XML into Neo4jGraph store"); |
70 | 78 |
loader.load(rdfFile); |
71 | 79 |
logger.info("loading RDF/XML done"); |
72 | 80 |
logger.info("Neo4jGraph has " + sizeInfo()); |
73 | 81 |
|
74 | 82 |
logger.info("rdf loaded into Neo4jGraph store at " + storeLocation); |
75 |
connection = new SailRepository(sail).getConnection(); |
|
76 | 83 |
} |
77 | 84 |
|
78 | 85 |
|
... | ... | |
91 | 98 |
} |
92 | 99 |
|
93 | 100 |
|
94 |
/** |
|
95 |
* @return the connection |
|
96 |
*/ |
|
97 |
public RepositoryConnectionBase connection() { |
|
98 |
return connection; |
|
99 |
} |
|
100 |
|
|
101 |
|
|
102 | 101 |
/** |
103 | 102 |
* {@inheritDoc} |
104 | 103 |
*/ |
... | ... | |
111 | 110 |
return graph; |
112 | 111 |
} |
113 | 112 |
|
113 |
/** |
|
114 |
* @return the sailRepo |
|
115 |
*/ |
|
116 |
public SailRepository getSailRepo() { |
|
117 |
return sailRepo; |
|
118 |
} |
|
114 | 119 |
|
115 | 120 |
|
116 | 121 |
} |
src/main/java/org/bgbm/biovel/drf/store/Store.java | ||
---|---|---|
45 | 45 |
storeLocation = new File(utisHome, storeName() + File.separator); |
46 | 46 |
if( !storeLocation.exists()) { |
47 | 47 |
storeLocation.mkdirs(); |
48 |
logger.debug("new store location created"); |
|
48 | 49 |
} |
49 | 50 |
initStoreEngine(); |
50 | 51 |
} |
... | ... | |
108 | 109 |
* the location of the file to load the rdf triples from |
109 | 110 |
* @throws Exception |
110 | 111 |
*/ |
111 |
public void loadIntoStore(String rdfFileUri) throws Exception { |
|
112 |
File rdfFile = downloadAndExtract(rdfFileUri);
|
|
113 |
clear(); |
|
112 |
public void loadIntoStore(String ... rdfFileUri) throws Exception {
|
|
113 |
stopStoreEngine();
|
|
114 |
// clear();
|
|
114 | 115 |
initStoreEngine(); |
115 |
load(rdfFile); |
|
116 |
for (String uri : rdfFileUri) { |
|
117 |
File localF = downloadAndExtract(uri); |
|
118 |
load(localF); |
|
119 |
} |
|
116 | 120 |
} |
117 | 121 |
|
118 | 122 |
protected abstract void initStoreEngine() throws Exception; |
119 | 123 |
|
124 |
protected abstract void stopStoreEngine() throws Exception; |
|
125 |
|
|
120 | 126 |
protected abstract void load(File rdfFile) throws Exception; |
121 | 127 |
|
122 | 128 |
protected void clear() throws IOException { |
src/main/java/org/bgbm/biovel/drf/store/TDBStore.java | ||
---|---|---|
110 | 110 |
this.dataset = dataset; |
111 | 111 |
} |
112 | 112 |
|
113 |
/** |
|
114 |
* {@inheritDoc} |
|
115 |
*/ |
|
116 |
@Override |
|
117 |
protected void stopStoreEngine() throws Exception { |
|
118 |
// TODO Auto-generated method stub |
|
119 |
|
|
120 |
} |
|
121 |
|
|
113 | 122 |
} |
src/main/java/org/bgbm/biovel/drf/utils/Profiler.java | ||
---|---|---|
1 |
// $Id$ |
|
2 |
/** |
|
3 |
* Copyright (C) 2015 EDIT |
|
4 |
* European Distributed Institute of Taxonomy |
|
5 |
* http://www.e-taxonomy.eu |
|
6 |
* |
|
7 |
* The contents of this file are subject to the Mozilla Public License Version 1.1 |
|
8 |
* See LICENSE.TXT at the top of this package for the full license terms. |
|
9 |
*/ |
|
10 |
package org.bgbm.biovel.drf.utils; |
|
11 |
|
|
12 |
import java.io.IOException; |
|
13 |
import java.io.PrintStream; |
|
14 |
|
|
15 |
import com.yourkit.api.Controller; |
|
16 |
|
|
17 |
/** |
|
18 |
* @author a.kohlbecker |
|
19 |
* @date Oct 22, 2015 |
|
20 |
* |
|
21 |
*/ |
|
22 |
public class Profiler { |
|
23 |
|
|
24 |
private Controller yourkit; |
|
25 |
private boolean yjp = false; |
|
26 |
private boolean cpu = false; |
|
27 |
private long start; |
|
28 |
|
|
29 |
private Profiler() { |
|
30 |
|
|
31 |
} |
|
32 |
|
|
33 |
public static Profiler newCpuProfiler(boolean yjp) { |
|
34 |
Profiler p = new Profiler(); |
|
35 |
p.yjp = yjp; |
|
36 |
p.cpu = true; |
|
37 |
if(yjp) { |
|
38 |
// https://www.yourkit.com/docs/java/api/com/yourkit/api/Controller.html |
|
39 |
try { |
|
40 |
p.yourkit = new Controller(); |
|
41 |
p.yourkit.enableStackTelemetry(); |
|
42 |
p.yourkit.startCPUTracing(null); |
|
43 |
} catch (Exception e) { |
|
44 |
e.printStackTrace(); |
|
45 |
} |
|
46 |
} |
|
47 |
|
|
48 |
p.start = System.currentTimeMillis(); |
|
49 |
|
|
50 |
return p; |
|
51 |
} |
|
52 |
|
|
53 |
public void end(PrintStream err) { |
|
54 |
|
|
55 |
Long time = System.currentTimeMillis() - start; |
|
56 |
|
|
57 |
try { |
|
58 |
err.append(time.toString()).append(" ms\n"); |
|
59 |
if(yjp) { |
|
60 |
if(cpu) { |
|
61 |
yourkit.stopCPUProfiling(); |
|
62 |
String fileLocation = yourkit.captureSnapshot(com.yourkit.api.Controller.SNAPSHOT_WITHOUT_HEAP); |
|
63 |
err.append("Snapshot stored at: ").append(fileLocation).append("\n"); |
|
64 |
} |
|
65 |
} |
|
66 |
} catch (IOException e) { |
|
67 |
e.printStackTrace(); |
|
68 |
} catch (Exception e) { |
|
69 |
e.printStackTrace(); |
|
70 |
} |
|
71 |
} |
|
72 |
} |
src/test/resources/log4j.properties | ||
---|---|---|
18 | 18 |
# for multi-threaded applications: |
19 | 19 |
log4j.appender.stdout.layout.ConversionPattern=%d %-5p [%t] %C{2} (%F:%L) - %m%n |
20 | 20 |
|
21 |
log4j.rootLogger=warn, stdout
|
|
21 |
log4j.rootLogger=info, stdout
|
|
22 | 22 |
|
23 | 23 |
log4j.logger.org.apache.http.impl.conn = info |
24 | 24 |
log4j.logger.org.apache.http.impl.conn.Wire = info |
Also available in: Unified diff
first workking implementation of the tinkerpop client for EEA_BDC