Project

General

Profile

« Previous | Next » 

Revision d5bca6b4

Added by Andreas Müller over 2 years ago

ref #9889 add direct matching on persistence to deduplication helper and replace for reference.author and .inReference

View differences:

cdmlib-io/src/test/java/eu/etaxonomy/cdm/io/referenceris/in/RisReferenceImportTest.java
13 13
import java.io.FileNotFoundException;
14 14
import java.io.IOException;
15 15
import java.net.URL;
16
import java.util.Collections;
16 17
import java.util.List;
17 18

  
18 19
import org.junit.Assert;
......
22 23
import org.unitils.spring.annotation.SpringBeanByName;
23 24
import org.unitils.spring.annotation.SpringBeanByType;
24 25

  
26
import eu.etaxonomy.cdm.api.service.IAgentService;
25 27
import eu.etaxonomy.cdm.api.service.IReferenceService;
26 28
//import eu.etaxonomy.cdm.common.DOI;
27 29
import eu.etaxonomy.cdm.io.common.CdmApplicationAwareDefaultImport;
28 30
import eu.etaxonomy.cdm.io.common.ImportResult;
29 31
import eu.etaxonomy.cdm.io.reference.ris.in.RisReferenceImportConfigurator;
32
import eu.etaxonomy.cdm.model.agent.Institution;
30 33
import eu.etaxonomy.cdm.model.agent.Person;
31 34
import eu.etaxonomy.cdm.model.agent.Team;
32 35
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
33 36
import eu.etaxonomy.cdm.model.common.CdmBase;
37
import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
34 38
import eu.etaxonomy.cdm.model.common.VerbatimTimePeriod;
35 39
import eu.etaxonomy.cdm.model.reference.Reference;
36 40
import eu.etaxonomy.cdm.model.reference.ReferenceType;
......
48 52
	@SpringBeanByType
49 53
	private IReferenceService referenceService;
50 54

  
51
	private RisReferenceImportConfigurator configurator;
52
    private RisReferenceImportConfigurator configLong;
55
	@SpringBeanByType
56
    private IAgentService agentService;
53 57

  
54 58
	@Before
55
	public void setUp() {
56
		String inputFile = "/eu/etaxonomy/cdm/io/reference/ris/in/RisReferenceImportTest-input.ris";
57

  
58
        try {
59
            URL url = this.getClass().getResource(inputFile);
60
            assertNotNull("URL for the test file '" + inputFile + "' does not exist", url);
61

  
62
            String inputFileLong = "/eu/etaxonomy/cdm/io/reference/ris/in/Acantholimon.ris";
63
            URL urlLong = this.getClass().getResource(inputFileLong);
64
            assertNotNull("URL for the test file '" + inputFileLong + "' does not exist", urlLong);
65

  
66
			configurator = RisReferenceImportConfigurator.NewInstance(url, null);
67
			configLong = RisReferenceImportConfigurator.NewInstance(urlLong, null);
68

  
69
		} catch (Exception e) {
70
			e.printStackTrace();
71
			Assert.fail();
72
		}
73
		assertNotNull("Configurator could not be created", configurator);
74
	    assertNotNull("Configurator could not be created", configLong);
75
	    assertNotNull("nameService should not be null", referenceService);
76
	}
59
	public void setUp() {}
77 60

  
78 61
//***************************** TESTS *************************************//
79 62

  
......
81 64
	@DataSet( value="/eu/etaxonomy/cdm/database/ClearDBDataSet.xml", loadStrategy=CleanSweepInsertLoadStrategy.class)
82 65
	//@Ignore
83 66
    public void testShort() {
84

  
67
	    RisReferenceImportConfigurator configurator = getConfigurator("RisReferenceImportTest-input.ris");
85 68
		ImportResult result = defaultImport.invoke(configurator);
86 69
		String report = result.createReport().toString();
87 70
		Assert.assertTrue(report.length() > 0);
......
142 125
	}
143 126

  
144 127
	@Test
145
	public void testChapter() throws IOException{
146
        String inputFileLong = "/eu/etaxonomy/cdm/io/reference/ris/in/Arias2012.ris";
147
        URL urlLong = this.getClass().getResource(inputFileLong);
148
        configurator = RisReferenceImportConfigurator.NewInstance(urlLong, null);
128
	public void testChapter() {
129

  
130
	    final RisReferenceImportConfigurator configurator = getConfigurator("Arias2012.ris");
149 131

  
150 132
        ImportResult result = defaultImport.invoke(configurator);
151 133
        String report = result.createReport().toString();
......
155 137
        Integer expected = 2;
156 138
        Assert.assertEquals(expected, result.getNewRecords(Reference.class));
157 139

  
158
        List<Reference> list = referenceService.list(Reference.class, null, null, null, null);
140
        List<Reference> referenceList = referenceService.list(Reference.class, null, null, null, null);
159 141
        Assert.assertEquals("There should be 3 references, the book-section, the book and the source reference",
160
                3, list.size());
142
                3, referenceList.size());
161 143

  
162 144
        //book section
163
        Reference bookSection = list.stream().filter(r->r.getType() == ReferenceType.BookSection).findFirst().get();
145
        Reference bookSection = referenceList.stream().filter(r->r.getType() == ReferenceType.BookSection).findFirst().get();
164 146
        //... title
165 147
        Assert.assertEquals("Cactaceae", bookSection.getTitle());
166 148
        //... author
......
182 164
        Assert.assertEquals("1-235", bookSection.getPages());
183 165

  
184 166
        //book
185
        Reference book = list.stream().filter(r->r.getType() == ReferenceType.Book).findFirst().get();
167
        Reference book = referenceList.stream().filter(r->r.getType() == ReferenceType.Book).findFirst().get();
186 168
        //... title
187 169
        Assert.assertEquals("Flora del Valle de Tehuac\u00E1n-Cuicatl\u00E1n", book.getTitle());
188 170
        Assert.assertEquals("Fasc\u00EDculo 95", book.getVolume());
......
190 172
        Assert.assertEquals("Instituto de Biolog\u00EDa, Universidad Nacional Aut\u00F3noma de M\u00E9xico", book.getPublisher());
191 173

  
192 174
        //source reference
193
        Reference sourceRef = list.stream().filter(r->r.equals(configurator.getSourceReference())).findFirst().get();
175
        Reference sourceRef = referenceList.stream().filter(r->r.equals(configurator.getSourceReference())).findFirst().get();
194 176
        Assert.assertNotNull(sourceRef);
195 177
        //TODO cont.
178

  
179
        List<Person> personList = agentService.list(Person.class, null, null, null, null);
180
        Assert.assertEquals("There should be 5 persons", 5, personList.size());
181

  
182
        List<Team> teamList = agentService.list(Team.class, null, null, null, null);
183
        Assert.assertEquals("There should be 1 team", 1, teamList.size());
184

  
185

  
186
        //test deduplication by running it again
187
        result = defaultImport.invoke(configurator);
188
        report = result.createReport().toString();
189
        Assert.assertTrue(report.contains("Reference: 0"));
190
        Assert.assertEquals(0, result.getErrors().size() + result.getExceptions().size() + result.getWarnings().size());
191
        referenceList = referenceService.list(Reference.class, null, null, null, null);
192
        Assert.assertEquals("There should still be 3 references, the book-section, the book and the source reference",
193
                3, referenceList.size());
194

  
195
        personList = agentService.list(Person.class, null, null, null, null);
196
        Assert.assertEquals("There should still be 5 persons", 5, personList.size());
197

  
198
        teamList = agentService.list(Team.class, null, null, null, null);
199
        Assert.assertEquals("There should still be 1 team", 1, teamList.size());
200

  
201
        //test deduplication by running another chapter
202
        RisReferenceImportConfigurator configurator2 = getConfigurator("Arias2012_2.ris");
203
        result = defaultImport.invoke(configurator2);
204
        report = result.createReport().toString();
205
//        Assert.assertTrue(report.contains("Reference: 0"));
206
        Assert.assertEquals(0, result.getErrors().size() + result.getExceptions().size() + result.getWarnings().size());
207
        referenceList = referenceService.list(Reference.class, null, null, null, null);
208
        Assert.assertEquals("There should be 5 references, 2 book-sections, the book and 2 source references",
209
                5, referenceList.size());
210

  
211
        personList = agentService.list(Person.class, null, null, null, null);
212
        Assert.assertEquals("There should be 6 persons now", 6, personList.size());
213

  
214
        teamList = agentService.list(Team.class, null, null, null, null);
215
        Assert.assertEquals("There should be 2 teams now", 2, teamList.size());
216

  
196 217
	}
197 218

  
198
	@Test
199
	//@Ignore
200
    public void testLongFile() {
201
        ImportResult result = defaultImport.invoke(configLong);
202
        String report = result.createReport().toString();
203
        System.out.println(report);
219
    private RisReferenceImportConfigurator getConfigurator(String fileName) {
220
        String inputFile = "/eu/etaxonomy/cdm/io/reference/ris/in/" + fileName;
221
        URL url = this.getClass().getResource(inputFile);
222
        assertNotNull("URL for the test file '" + inputFile + "' does not exist", url);
223
        try {
224
            RisReferenceImportConfigurator result = RisReferenceImportConfigurator.NewInstance(url, null);
225
            result.setDeduplicationMaxCountForFullLoad(1);
226
            return result;
227
        } catch (IOException e) {
228
            Assert.fail("IOException while creating configurator: " + e.getMessage());
229
            return null;
230
        }
231
    }
204 232

  
205
        Integer expected = 118;  //did not count yet
206
        Assert.assertEquals(expected, result.getNewRecords(Reference.class));
233
    @Test
234
    public void testLongFile() {
207 235

  
208
        List<Reference> list = referenceService.list(Reference.class, null, null, null, null);
209
//        Assert.assertEquals("There should be 119 references (still need to count them)", 119, list.size());
210
        //TODO deduplication
236
        RisReferenceImportConfigurator configurator = getConfigurator("Acantholimon.ris");
237
        ImportResult result = defaultImport.invoke(configurator);
211 238

  
212
        Reference ref58 = list.stream().filter(r->hasId(r, "58", false)).findFirst().get();
239
        @SuppressWarnings("unused")
240
        String report = result.createReport().toString();
241
//        System.out.println(report);
242

  
243
//        Integer expectedWithoutDeduplication = 118;  //did not count yet
244
        Integer expectedDeduplicated = 104;  //did not count yet
245
        Assert.assertEquals(expectedDeduplicated, result.getNewRecords(Reference.class));
246
//        System.out.println("Person: "+ result.getNewRecords(Person.class));
247
//        System.out.println("Team: "+ result.getNewRecords(Team.class));
248

  
249
        List<Reference> refList = referenceService.list(Reference.class, null, null, null, null);
250
//        Assert.assertEquals("There should be 119 references (still need to count them)", 119, refList.size());
251
        Collections.sort(refList, (r1,r2) -> r1.getTitleCache().compareTo(r2.getTitleCache()));
252
        printList(refList);
253
        List<Person> personList = agentService.list(Person.class, null, null, null, null);
254
        printList(personList);
255
        Assert.assertEquals(99, personList.size());
256
        List<Team> teamList = agentService.list(Team.class, null, null, null, null);
257
        printList(teamList);
258
        Assert.assertEquals(33, teamList.size());
259
        List<Institution> institutionList = agentService.list(Institution.class, null, null, null, null);
260
        printList(institutionList);
261
        Assert.assertEquals(0, institutionList.size());
262

  
263

  
264
        Reference ref58 = refList.stream().filter(r->hasId(r, "58", false)).findFirst().get();
213 265
        Assert.assertNotNull("", ref58);
214 266
        Assert.assertEquals((Integer)2003, ref58.getDatePublished().getStartYear());
215 267

  
216
        Reference ref53 = list.stream().filter(r->hasId(r, "53", false)).findFirst().get();
268
        Reference ref53 = refList.stream().filter(r->hasId(r, "53", false)).findFirst().get();
217 269
        Assert.assertNotNull("", ref53);
218 270
        Assert.assertEquals(ReferenceType.BookSection, ref53.getType());
219 271
        Assert.assertNotNull("", ref53.getInReference());
......
265 317
//        }
266 318
    }
267 319

  
320
    private void printList(List<? extends IdentifiableEntity<?>> list) {
321
        if (!logger.isDebugEnabled()){
322
            return;
323
        }
324
        System.out.println(list.size());
325
        Collections.sort(list, (p1,p2) -> p1.getTitleCache().compareTo(p2.getTitleCache()));
326
        list.stream().forEach(r->System.out.println(r.getTitleCache()));
327
    }
328

  
268 329
    private boolean hasId(Reference ref, String idStr, boolean getInRef) {
269 330
        if (ref.getSources().size() != 1){
270 331
            return false;

Also available in: Unified diff