Project

General

Profile

Revision 95b8352b

ID95b8352b0776b263bb05d88f0ca861bf2227a8e5
Parent 898b8fa6
Child 49b1f2d4

Added by Patrick Plitzner 6 months ago

ref #8248 Enhance OWL import

  • Add term URI to import
  • add sources to imported terms
  • create term vocabulary for imported terms
  • check for already existing terms
  • save terms, feature nodes and feature trees to DB
  • update OWL import test with added properties

View differences:

cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/descriptive/owl/in/OwlImport.java
10 10

  
11 11
import java.net.URI;
12 12
import java.util.ArrayList;
13
import java.util.Collection;
13 14
import java.util.List;
14 15

  
15 16
import org.springframework.stereotype.Component;
......
23 24

  
24 25
import eu.etaxonomy.cdm.io.common.CdmImportBase;
25 26
import eu.etaxonomy.cdm.io.descriptive.owl.OwlConstants;
27
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
26 28
import eu.etaxonomy.cdm.model.description.Feature;
27 29
import eu.etaxonomy.cdm.model.term.DefinedTerm;
28 30
import eu.etaxonomy.cdm.model.term.DefinedTermBase;
29 31
import eu.etaxonomy.cdm.model.term.FeatureNode;
30 32
import eu.etaxonomy.cdm.model.term.FeatureTree;
31 33
import eu.etaxonomy.cdm.model.term.TermType;
34
import eu.etaxonomy.cdm.model.term.TermVocabulary;
35
import eu.etaxonomy.cdm.persistence.dto.TermDto;
32 36

  
33 37
/**
34 38
 * @author pplitzner
......
40 44

  
41 45
    private static final long serialVersionUID = -3659780404413458511L;
42 46

  
47
    private static final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(OwlImport.class);
48

  
43 49
    private Property propertyHasRootNode;
44 50
    private Property propHasSubStructure;
45 51
    private Property propUuid;
52
    private Property propUri;
46 53
    private Property propLabel;
47 54
    private Property propIsA;
48 55
    private Property propType;
......
61 68
        propertyHasRootNode = model.createProperty(OwlConstants.PROPERTY_HAS_ROOT_NODE);
62 69
        propHasSubStructure = model.createProperty(OwlConstants.PROPERTY_HAS_SUBSTRUCTURE);
63 70
        propUuid = model.createProperty(OwlConstants.PROPERTY_UUID);
71
        propUri = model.createProperty(OwlConstants.PROPERTY_URI);
64 72
        propLabel = model.createProperty(OwlConstants.PROPERTY_LABEL);
65 73
        propIsA = model.createProperty(OwlConstants.PROPERTY_IS_A);
66 74
        propType = model.createProperty(OwlConstants.PROPERTY_TYPE);
......
78 86
            featureTree.setTitleCache(tree.getProperty(propLabel).getString(), true);
79 87

  
80 88
            Resource rootNode = tree.getProperty(propertyHasRootNode).getResource();
81
            rootNode.listProperties(propHasSubStructure).forEachRemaining(prop->createNode(featureTree.getRoot(), prop, model));
89
            rootNode.listProperties(propHasSubStructure).forEachRemaining(prop->createNode(featureTree.getRoot(), prop, featureTree.getTitleCache(), model, state));
82 90

  
83 91
            featureTrees.add(featureTree);
92

  
93
            getFeatureTreeService().save(featureTree);
84 94
        }
85
        state.setFeatureTrees(featureTrees);
86 95
    }
87 96

  
88
    private void createNode(FeatureNode parent, Statement nodeStatement, Model model) {
97
    private void createNode(FeatureNode parent, Statement nodeStatement, String treeLabel, Model model, OwlImportState state) {
89 98
        Resource nodeResource = model.createResource(OwlConstants.RESOURCE_NODE+nodeStatement.getProperty(propUuid).getString());
90 99

  
91 100
        String termLabel = nodeResource.getProperty(propLabel).getString();
92 101
        String termDescription = nodeResource.hasProperty(propDescription)?nodeResource.getProperty(propDescription).getString():null;
93
        String termType = nodeResource.getProperty(propType).getString();
94
        DefinedTermBase term;
95
        if(termType.equals(TermType.Feature)){
96
            term = Feature.NewInstance();
102
        TermType termType = TermType.getByKey(nodeResource.getProperty(propType).getString());
103
        String uriString = nodeResource.hasProperty(propUri)?nodeResource.getProperty(propUri).toString():null;
104

  
105
        Collection<TermDto> dtos = new ArrayList<>();
106
        if(uriString!=null){
107
            URI uri = URI.create(uriString);
108
            dtos = getTermService().findByUriAsDto(uri, termLabel, termType);
97 109
        }
98 110
        else{
99
            term = DefinedTerm.NewInstance(TermType.getByKey(termType), termDescription, termLabel, null);
111
            dtos = getTermService().findByTitleAsDto(termLabel, termType);
112
        }
113
        DefinedTermBase term = null;
114
        if(dtos.size()>=1){
115
            logger.warn("More than one term was found for: "+termLabel+"\nUsing the first one found");
116
            term = getTermService().load(dtos.iterator().next().getUuid());
117
        }
118
        if(term==null){
119
            if(termType.equals(TermType.Feature)){
120
                term = Feature.NewInstance();
121
            }
122
            else{
123
                term = DefinedTerm.NewInstance(termType, termDescription, termLabel, null);
124
            }
125
            IdentifiableSource importSource = IdentifiableSource.NewDataImportInstance(termLabel);
126
            importSource.setCitation(state.getConfig().getSourceReference());
127
            term.addSource(importSource);
128
            getTermService().save(term);
129

  
130
            TermVocabulary vocabulary = state.getConfig().getVocabulary(termType, treeLabel);
131
            vocabulary.addTerm(term);
132
            getVocabularyService().saveOrUpdate(vocabulary);
100 133
        }
101 134

  
102 135
        FeatureNode<?> childNode = FeatureNode.NewInstance(term);
103 136
        parent.addChild(childNode);
104 137

  
105
        nodeResource.listProperties(propHasSubStructure).forEachRemaining(prop->createNode(childNode, prop, model));
138
        nodeResource.listProperties(propHasSubStructure).forEachRemaining(prop->createNode(childNode, prop, treeLabel, model, state));
106 139
    }
107 140

  
108 141
    @Override
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/descriptive/owl/in/OwlImportConfigurator.java
13 13
import eu.etaxonomy.cdm.io.common.ImportConfiguratorBase;
14 14
import eu.etaxonomy.cdm.io.common.ImportStateBase;
15 15
import eu.etaxonomy.cdm.model.reference.Reference;
16
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
17
import eu.etaxonomy.cdm.model.term.TermType;
18
import eu.etaxonomy.cdm.model.term.TermVocabulary;
16 19

  
17 20
/**
18 21
 * @author pplitzner
......
23 26

  
24 27
    private static final long serialVersionUID = -7981427548996602252L;
25 28

  
29
    private TermVocabulary vocabulary;
30

  
26 31
    public static OwlImportConfigurator NewInstance(URI source){
27 32
        return new OwlImportConfigurator(source);
28 33
    }
......
30 35
    protected OwlImportConfigurator(URI source) {
31 36
        super(null);
32 37
        this.setSource(source);
38
        Reference reference = ReferenceFactory.newGeneric();
39
        reference.setTitle("OWL import from "+source);
40
        this.setSourceReference(reference);
33 41
    }
34 42

  
35 43
    @Override
......
46 54

  
47 55
    @Override
48 56
    public Reference getSourceReference() {
49
        return null;
57
        return sourceReference;
58
    }
59

  
60
    public TermVocabulary getVocabulary(TermType termType, String vocLabel) {
61
        if(vocabulary==null){
62
            vocabulary = TermVocabulary.NewInstance(termType);
63
            vocabulary.setLabel(vocLabel);
64
        }
65
        return vocabulary;
50 66
    }
51 67

  
52 68
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/descriptive/owl/in/OwlImportState.java
8 8
*/
9 9
package eu.etaxonomy.cdm.io.descriptive.owl.in;
10 10

  
11
import java.util.Collection;
12

  
13 11
import eu.etaxonomy.cdm.io.common.ImportStateBase;
14
import eu.etaxonomy.cdm.model.term.FeatureTree;
15 12

  
16 13
/**
17 14
 * @author pplitzner
......
20 17
 */
21 18
public class OwlImportState extends ImportStateBase<OwlImportConfigurator, OwlImport> {
22 19

  
23
    private Collection<FeatureTree> featureTrees;
24

  
25 20
    public OwlImportState(OwlImportConfigurator config) {
26 21
        super(config);
27 22
    }
28 23

  
29
    public Collection<FeatureTree> getFeatureTrees() {
30
        return featureTrees;
31
    }
32

  
33
    public void setFeatureTrees(Collection<FeatureTree> featureTrees) {
34
        this.featureTrees = featureTrees;
35
    }
36

  
37 24
}
cdmlib-io/src/test/java/eu/etaxonomy/cdm/io/owl/out/in/OwlImportTest.java
9 9

  
10 10
package eu.etaxonomy.cdm.io.owl.out.in;
11 11

  
12
import static org.junit.Assert.assertEquals;
12 13
import static org.junit.Assert.assertNotNull;
13 14
import static org.junit.Assert.assertTrue;
14 15

  
......
16 17
import java.net.URI;
17 18
import java.net.URISyntaxException;
18 19
import java.net.URL;
19
import java.util.Collection;
20
import java.util.ArrayList;
21
import java.util.Arrays;
20 22
import java.util.List;
21 23

  
22 24
import org.junit.Before;
......
24 26
import org.unitils.dbunit.annotation.DataSet;
25 27
import org.unitils.spring.annotation.SpringBeanByType;
26 28

  
29
import eu.etaxonomy.cdm.api.service.IFeatureTreeService;
30
import eu.etaxonomy.cdm.api.service.ITermService;
31
import eu.etaxonomy.cdm.api.service.IVocabularyService;
27 32
import eu.etaxonomy.cdm.common.CdmUtils;
28 33
import eu.etaxonomy.cdm.io.descriptive.owl.in.OwlImport;
29 34
import eu.etaxonomy.cdm.io.descriptive.owl.in.OwlImportConfigurator;
......
32 37
import eu.etaxonomy.cdm.model.term.FeatureNode;
33 38
import eu.etaxonomy.cdm.model.term.FeatureTree;
34 39
import eu.etaxonomy.cdm.model.term.TermType;
40
import eu.etaxonomy.cdm.model.term.TermVocabulary;
41
import eu.etaxonomy.cdm.persistence.query.MatchMode;
35 42
import eu.etaxonomy.cdm.test.integration.CdmTransactionalIntegrationTest;
36 43

  
37 44
/**
......
45 52
    @SpringBeanByType
46 53
    private OwlImport owlImport;
47 54

  
55
    @SpringBeanByType
56
    private ITermService termService;
57

  
58
    @SpringBeanByType
59
    private IFeatureTreeService featureTreeService;
60

  
61
    @SpringBeanByType
62
    private IVocabularyService vocabularyService;
63

  
48 64
    private OwlImportConfigurator configurator;
49 65

  
66
    private FeatureTree tree;
67

  
50 68
    @Before
51 69
    public void setUp() throws URISyntaxException {
52 70
        URL url = this.getClass().getResource("/eu/etaxonomy/cdm/io/owl/in/test_structures.owl");
......
68 86
        this.setComplete();
69 87
        this.endTransaction();
70 88

  
71
        Collection<FeatureTree> featureTrees = state.getFeatureTrees();
72
        assertTrue("Wrong number of feature trees imported", featureTrees.size()==1);
73
        FeatureTree tree = featureTrees.iterator().next();
74
        assertTrue("Tree has wrong term type", tree.getTermType().equals(TermType.Structure));
75
        assertTrue("Wrong number of distinct features", tree.getDistinctFeatures().size()==4);
89
        String treeLabel = "TestStructures";
90
        List<FeatureTree> trees = featureTreeService.listByTitle(FeatureTree.class, treeLabel, MatchMode.EXACT, null, null, null, null, null);
91
        List<String> nodeProperties = new ArrayList<>();
92
        nodeProperties.add("term");
93
        tree = featureTreeService.loadWithNodes(trees.iterator().next().getUuid(), null, nodeProperties);
94
        assertNotNull("featureTree should not be null", tree);
95

  
96
        assertEquals("Tree has wrong term type", TermType.Structure, tree.getTermType());
97
        assertEquals("Wrong number of distinct features", 4, tree.getDistinctFeatures().size());
76 98
        List rootChildren = tree.getRootChildren();
77
        assertTrue("Wrong number of root children", rootChildren.size()==1);
99
        assertEquals("Wrong number of root children", 1, rootChildren.size());
78 100
        Object root = rootChildren.iterator().next();
79 101
        assertTrue("Root is no feature node", root instanceof FeatureNode);
80
        assertTrue("Root node has wrong term type", ((FeatureNode)root).getTermType().equals(TermType.Structure));
102
        assertEquals("Root node has wrong term type", TermType.Structure, ((FeatureNode)root).getTermType());
81 103
        FeatureNode<DefinedTerm> rootNode = (FeatureNode<DefinedTerm>) root;
82 104
        List<FeatureNode<DefinedTerm>> childNodes = rootNode.getChildNodes();
83
        assertTrue("Wrong number of children", childNodes.size()==2);
105
        assertEquals("Wrong number of children", 2, childNodes.size());
106
        DefinedTerm inflorescence = null;
84 107
        for (FeatureNode<DefinedTerm> featureNode : childNodes) {
85 108
            assertTrue("Child node not found",
86 109
                    featureNode.getTerm().getLabel().equals("inflorescence")
87 110
                    || featureNode.getTerm().getLabel().equals("Flower"));
88 111
            if(featureNode.getTerm().getLabel().equals("inflorescence")){
89 112
                assertTrue("Description not found", CdmUtils.isNotBlank(featureNode.getTerm().getDescription()));
90
                assertTrue("Description wrong", featureNode.getTerm().getDescription().equals(" the part of the plant that bears the flowers, including all its bracts  branches and flowers  but excluding unmodified leaves               "));
113
                String expectedDescription = " the part of the plant that bears the flowers, including all its bracts  branches and flowers  but excluding unmodified leaves               ";
114
                assertEquals("Description wrong", expectedDescription, featureNode.getTerm().getDescription());
115
                inflorescence = featureNode.getTerm();
91 116
            }
92 117
        }
118
        assertNotNull("term is null", inflorescence);
119
        assertEquals("Wrong term type", TermType.Structure, inflorescence.getTermType());
120

  
121
        List<TermVocabulary> vocs = vocabularyService.findByTitle(TermVocabulary.class, treeLabel, MatchMode.EXACT, null, null, null, null, Arrays.asList("terms")).getRecords();
122
        assertEquals("wrong number of vocabularies", 1, vocs.size());
123
        TermVocabulary termVoc = vocs.iterator().next();
124
        assertEquals("Wrong vocabulary label", treeLabel, termVoc.getTitleCache());
125
        assertEquals(4, termVoc.getTerms().size());
126
        assertTrue("Term not included in vocabulary", termVoc.getTerms().contains(inflorescence));
93 127

  
94 128
    }
95 129

  

Also available in: Unified diff

Add picture from clipboard (Maximum size: 40 MB)