Revision d5bca6b4
Added by Andreas Müller over 2 years ago
cdmlib-io/src/test/java/eu/etaxonomy/cdm/io/referenceris/in/RisReferenceImportTest.java | ||
---|---|---|
13 | 13 |
import java.io.FileNotFoundException; |
14 | 14 |
import java.io.IOException; |
15 | 15 |
import java.net.URL; |
16 |
import java.util.Collections; |
|
16 | 17 |
import java.util.List; |
17 | 18 |
|
18 | 19 |
import org.junit.Assert; |
... | ... | |
22 | 23 |
import org.unitils.spring.annotation.SpringBeanByName; |
23 | 24 |
import org.unitils.spring.annotation.SpringBeanByType; |
24 | 25 |
|
26 |
import eu.etaxonomy.cdm.api.service.IAgentService; |
|
25 | 27 |
import eu.etaxonomy.cdm.api.service.IReferenceService; |
26 | 28 |
//import eu.etaxonomy.cdm.common.DOI; |
27 | 29 |
import eu.etaxonomy.cdm.io.common.CdmApplicationAwareDefaultImport; |
28 | 30 |
import eu.etaxonomy.cdm.io.common.ImportResult; |
29 | 31 |
import eu.etaxonomy.cdm.io.reference.ris.in.RisReferenceImportConfigurator; |
32 |
import eu.etaxonomy.cdm.model.agent.Institution; |
|
30 | 33 |
import eu.etaxonomy.cdm.model.agent.Person; |
31 | 34 |
import eu.etaxonomy.cdm.model.agent.Team; |
32 | 35 |
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase; |
33 | 36 |
import eu.etaxonomy.cdm.model.common.CdmBase; |
37 |
import eu.etaxonomy.cdm.model.common.IdentifiableEntity; |
|
34 | 38 |
import eu.etaxonomy.cdm.model.common.VerbatimTimePeriod; |
35 | 39 |
import eu.etaxonomy.cdm.model.reference.Reference; |
36 | 40 |
import eu.etaxonomy.cdm.model.reference.ReferenceType; |
... | ... | |
48 | 52 |
@SpringBeanByType |
49 | 53 |
private IReferenceService referenceService; |
50 | 54 |
|
51 |
private RisReferenceImportConfigurator configurator;
|
|
52 |
private RisReferenceImportConfigurator configLong;
|
|
55 |
@SpringBeanByType
|
|
56 |
private IAgentService agentService;
|
|
53 | 57 |
|
54 | 58 |
@Before |
55 |
public void setUp() { |
|
56 |
String inputFile = "/eu/etaxonomy/cdm/io/reference/ris/in/RisReferenceImportTest-input.ris"; |
|
57 |
|
|
58 |
try { |
|
59 |
URL url = this.getClass().getResource(inputFile); |
|
60 |
assertNotNull("URL for the test file '" + inputFile + "' does not exist", url); |
|
61 |
|
|
62 |
String inputFileLong = "/eu/etaxonomy/cdm/io/reference/ris/in/Acantholimon.ris"; |
|
63 |
URL urlLong = this.getClass().getResource(inputFileLong); |
|
64 |
assertNotNull("URL for the test file '" + inputFileLong + "' does not exist", urlLong); |
|
65 |
|
|
66 |
configurator = RisReferenceImportConfigurator.NewInstance(url, null); |
|
67 |
configLong = RisReferenceImportConfigurator.NewInstance(urlLong, null); |
|
68 |
|
|
69 |
} catch (Exception e) { |
|
70 |
e.printStackTrace(); |
|
71 |
Assert.fail(); |
|
72 |
} |
|
73 |
assertNotNull("Configurator could not be created", configurator); |
|
74 |
assertNotNull("Configurator could not be created", configLong); |
|
75 |
assertNotNull("nameService should not be null", referenceService); |
|
76 |
} |
|
59 |
public void setUp() {} |
|
77 | 60 |
|
78 | 61 |
//***************************** TESTS *************************************// |
79 | 62 |
|
... | ... | |
81 | 64 |
@DataSet( value="/eu/etaxonomy/cdm/database/ClearDBDataSet.xml", loadStrategy=CleanSweepInsertLoadStrategy.class) |
82 | 65 |
//@Ignore |
83 | 66 |
public void testShort() { |
84 |
|
|
67 |
RisReferenceImportConfigurator configurator = getConfigurator("RisReferenceImportTest-input.ris"); |
|
85 | 68 |
ImportResult result = defaultImport.invoke(configurator); |
86 | 69 |
String report = result.createReport().toString(); |
87 | 70 |
Assert.assertTrue(report.length() > 0); |
... | ... | |
142 | 125 |
} |
143 | 126 |
|
144 | 127 |
@Test |
145 |
public void testChapter() throws IOException{ |
|
146 |
String inputFileLong = "/eu/etaxonomy/cdm/io/reference/ris/in/Arias2012.ris"; |
|
147 |
URL urlLong = this.getClass().getResource(inputFileLong); |
|
148 |
configurator = RisReferenceImportConfigurator.NewInstance(urlLong, null); |
|
128 |
public void testChapter() { |
|
129 |
|
|
130 |
final RisReferenceImportConfigurator configurator = getConfigurator("Arias2012.ris"); |
|
149 | 131 |
|
150 | 132 |
ImportResult result = defaultImport.invoke(configurator); |
151 | 133 |
String report = result.createReport().toString(); |
... | ... | |
155 | 137 |
Integer expected = 2; |
156 | 138 |
Assert.assertEquals(expected, result.getNewRecords(Reference.class)); |
157 | 139 |
|
158 |
List<Reference> list = referenceService.list(Reference.class, null, null, null, null);
|
|
140 |
List<Reference> referenceList = referenceService.list(Reference.class, null, null, null, null);
|
|
159 | 141 |
Assert.assertEquals("There should be 3 references, the book-section, the book and the source reference", |
160 |
3, list.size());
|
|
142 |
3, referenceList.size());
|
|
161 | 143 |
|
162 | 144 |
//book section |
163 |
Reference bookSection = list.stream().filter(r->r.getType() == ReferenceType.BookSection).findFirst().get();
|
|
145 |
Reference bookSection = referenceList.stream().filter(r->r.getType() == ReferenceType.BookSection).findFirst().get();
|
|
164 | 146 |
//... title |
165 | 147 |
Assert.assertEquals("Cactaceae", bookSection.getTitle()); |
166 | 148 |
//... author |
... | ... | |
182 | 164 |
Assert.assertEquals("1-235", bookSection.getPages()); |
183 | 165 |
|
184 | 166 |
//book |
185 |
Reference book = list.stream().filter(r->r.getType() == ReferenceType.Book).findFirst().get();
|
|
167 |
Reference book = referenceList.stream().filter(r->r.getType() == ReferenceType.Book).findFirst().get();
|
|
186 | 168 |
//... title |
187 | 169 |
Assert.assertEquals("Flora del Valle de Tehuac\u00E1n-Cuicatl\u00E1n", book.getTitle()); |
188 | 170 |
Assert.assertEquals("Fasc\u00EDculo 95", book.getVolume()); |
... | ... | |
190 | 172 |
Assert.assertEquals("Instituto de Biolog\u00EDa, Universidad Nacional Aut\u00F3noma de M\u00E9xico", book.getPublisher()); |
191 | 173 |
|
192 | 174 |
//source reference |
193 |
Reference sourceRef = list.stream().filter(r->r.equals(configurator.getSourceReference())).findFirst().get();
|
|
175 |
Reference sourceRef = referenceList.stream().filter(r->r.equals(configurator.getSourceReference())).findFirst().get();
|
|
194 | 176 |
Assert.assertNotNull(sourceRef); |
195 | 177 |
//TODO cont. |
178 |
|
|
179 |
List<Person> personList = agentService.list(Person.class, null, null, null, null); |
|
180 |
Assert.assertEquals("There should be 5 persons", 5, personList.size()); |
|
181 |
|
|
182 |
List<Team> teamList = agentService.list(Team.class, null, null, null, null); |
|
183 |
Assert.assertEquals("There should be 1 team", 1, teamList.size()); |
|
184 |
|
|
185 |
|
|
186 |
//test deduplication by running it again |
|
187 |
result = defaultImport.invoke(configurator); |
|
188 |
report = result.createReport().toString(); |
|
189 |
Assert.assertTrue(report.contains("Reference: 0")); |
|
190 |
Assert.assertEquals(0, result.getErrors().size() + result.getExceptions().size() + result.getWarnings().size()); |
|
191 |
referenceList = referenceService.list(Reference.class, null, null, null, null); |
|
192 |
Assert.assertEquals("There should still be 3 references, the book-section, the book and the source reference", |
|
193 |
3, referenceList.size()); |
|
194 |
|
|
195 |
personList = agentService.list(Person.class, null, null, null, null); |
|
196 |
Assert.assertEquals("There should still be 5 persons", 5, personList.size()); |
|
197 |
|
|
198 |
teamList = agentService.list(Team.class, null, null, null, null); |
|
199 |
Assert.assertEquals("There should still be 1 team", 1, teamList.size()); |
|
200 |
|
|
201 |
//test deduplication by running another chapter |
|
202 |
RisReferenceImportConfigurator configurator2 = getConfigurator("Arias2012_2.ris"); |
|
203 |
result = defaultImport.invoke(configurator2); |
|
204 |
report = result.createReport().toString(); |
|
205 |
// Assert.assertTrue(report.contains("Reference: 0")); |
|
206 |
Assert.assertEquals(0, result.getErrors().size() + result.getExceptions().size() + result.getWarnings().size()); |
|
207 |
referenceList = referenceService.list(Reference.class, null, null, null, null); |
|
208 |
Assert.assertEquals("There should be 5 references, 2 book-sections, the book and 2 source references", |
|
209 |
5, referenceList.size()); |
|
210 |
|
|
211 |
personList = agentService.list(Person.class, null, null, null, null); |
|
212 |
Assert.assertEquals("There should be 6 persons now", 6, personList.size()); |
|
213 |
|
|
214 |
teamList = agentService.list(Team.class, null, null, null, null); |
|
215 |
Assert.assertEquals("There should be 2 teams now", 2, teamList.size()); |
|
216 |
|
|
196 | 217 |
} |
197 | 218 |
|
198 |
@Test |
|
199 |
//@Ignore |
|
200 |
public void testLongFile() { |
|
201 |
ImportResult result = defaultImport.invoke(configLong); |
|
202 |
String report = result.createReport().toString(); |
|
203 |
System.out.println(report); |
|
219 |
private RisReferenceImportConfigurator getConfigurator(String fileName) { |
|
220 |
String inputFile = "/eu/etaxonomy/cdm/io/reference/ris/in/" + fileName; |
|
221 |
URL url = this.getClass().getResource(inputFile); |
|
222 |
assertNotNull("URL for the test file '" + inputFile + "' does not exist", url); |
|
223 |
try { |
|
224 |
RisReferenceImportConfigurator result = RisReferenceImportConfigurator.NewInstance(url, null); |
|
225 |
result.setDeduplicationMaxCountForFullLoad(1); |
|
226 |
return result; |
|
227 |
} catch (IOException e) { |
|
228 |
Assert.fail("IOException while creating configurator: " + e.getMessage()); |
|
229 |
return null; |
|
230 |
} |
|
231 |
} |
|
204 | 232 |
|
205 |
Integer expected = 118; //did not count yet
|
|
206 |
Assert.assertEquals(expected, result.getNewRecords(Reference.class));
|
|
233 |
@Test
|
|
234 |
public void testLongFile() {
|
|
207 | 235 |
|
208 |
List<Reference> list = referenceService.list(Reference.class, null, null, null, null); |
|
209 |
// Assert.assertEquals("There should be 119 references (still need to count them)", 119, list.size()); |
|
210 |
//TODO deduplication |
|
236 |
RisReferenceImportConfigurator configurator = getConfigurator("Acantholimon.ris"); |
|
237 |
ImportResult result = defaultImport.invoke(configurator); |
|
211 | 238 |
|
212 |
Reference ref58 = list.stream().filter(r->hasId(r, "58", false)).findFirst().get(); |
|
239 |
@SuppressWarnings("unused") |
|
240 |
String report = result.createReport().toString(); |
|
241 |
// System.out.println(report); |
|
242 |
|
|
243 |
// Integer expectedWithoutDeduplication = 118; //did not count yet |
|
244 |
Integer expectedDeduplicated = 104; //did not count yet |
|
245 |
Assert.assertEquals(expectedDeduplicated, result.getNewRecords(Reference.class)); |
|
246 |
// System.out.println("Person: "+ result.getNewRecords(Person.class)); |
|
247 |
// System.out.println("Team: "+ result.getNewRecords(Team.class)); |
|
248 |
|
|
249 |
List<Reference> refList = referenceService.list(Reference.class, null, null, null, null); |
|
250 |
// Assert.assertEquals("There should be 119 references (still need to count them)", 119, refList.size()); |
|
251 |
Collections.sort(refList, (r1,r2) -> r1.getTitleCache().compareTo(r2.getTitleCache())); |
|
252 |
printList(refList); |
|
253 |
List<Person> personList = agentService.list(Person.class, null, null, null, null); |
|
254 |
printList(personList); |
|
255 |
Assert.assertEquals(99, personList.size()); |
|
256 |
List<Team> teamList = agentService.list(Team.class, null, null, null, null); |
|
257 |
printList(teamList); |
|
258 |
Assert.assertEquals(33, teamList.size()); |
|
259 |
List<Institution> institutionList = agentService.list(Institution.class, null, null, null, null); |
|
260 |
printList(institutionList); |
|
261 |
Assert.assertEquals(0, institutionList.size()); |
|
262 |
|
|
263 |
|
|
264 |
Reference ref58 = refList.stream().filter(r->hasId(r, "58", false)).findFirst().get(); |
|
213 | 265 |
Assert.assertNotNull("", ref58); |
214 | 266 |
Assert.assertEquals((Integer)2003, ref58.getDatePublished().getStartYear()); |
215 | 267 |
|
216 |
Reference ref53 = list.stream().filter(r->hasId(r, "53", false)).findFirst().get();
|
|
268 |
Reference ref53 = refList.stream().filter(r->hasId(r, "53", false)).findFirst().get();
|
|
217 | 269 |
Assert.assertNotNull("", ref53); |
218 | 270 |
Assert.assertEquals(ReferenceType.BookSection, ref53.getType()); |
219 | 271 |
Assert.assertNotNull("", ref53.getInReference()); |
... | ... | |
265 | 317 |
// } |
266 | 318 |
} |
267 | 319 |
|
320 |
private void printList(List<? extends IdentifiableEntity<?>> list) { |
|
321 |
if (!logger.isDebugEnabled()){ |
|
322 |
return; |
|
323 |
} |
|
324 |
System.out.println(list.size()); |
|
325 |
Collections.sort(list, (p1,p2) -> p1.getTitleCache().compareTo(p2.getTitleCache())); |
|
326 |
list.stream().forEach(r->System.out.println(r.getTitleCache())); |
|
327 |
} |
|
328 |
|
|
268 | 329 |
private boolean hasId(Reference ref, String idStr, boolean getInRef) { |
269 | 330 |
if (ref.getSources().size() != 1){ |
270 | 331 |
return false; |
Also available in: Unified diff
ref #9889 add direct matching on persistence to deduplication helper and replace for reference.author and .inReference