2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.iapt
;
12 import eu
.etaxonomy
.cdm
.api
.facade
.DerivedUnitFacade
;
13 import eu
.etaxonomy
.cdm
.common
.CdmUtils
;
14 import eu
.etaxonomy
.cdm
.io
.mexico
.SimpleExcelTaxonImport
;
15 import eu
.etaxonomy
.cdm
.io
.mexico
.SimpleExcelTaxonImportState
;
16 import eu
.etaxonomy
.cdm
.model
.agent
.Institution
;
17 import eu
.etaxonomy
.cdm
.model
.agent
.Person
;
18 import eu
.etaxonomy
.cdm
.model
.agent
.TeamOrPersonBase
;
19 import eu
.etaxonomy
.cdm
.model
.common
.*;
20 import eu
.etaxonomy
.cdm
.model
.name
.*;
21 import eu
.etaxonomy
.cdm
.model
.occurrence
.*;
22 import eu
.etaxonomy
.cdm
.model
.occurrence
.Collection
;
23 import eu
.etaxonomy
.cdm
.model
.reference
.Reference
;
24 import eu
.etaxonomy
.cdm
.model
.taxon
.*;
25 import eu
.etaxonomy
.cdm
.strategy
.parser
.NonViralNameParserImpl
;
26 import org
.apache
.commons
.lang
.ArrayUtils
;
27 import org
.apache
.commons
.lang
.StringEscapeUtils
;
28 import org
.apache
.commons
.lang
.StringUtils
;
29 import org
.apache
.log4j
.Level
;
30 import org
.apache
.log4j
.Logger
;
31 import org
.joda
.time
.DateTimeFieldType
;
32 import org
.joda
.time
.Partial
;
33 import org
.joda
.time
.format
.DateTimeFormat
;
34 import org
.joda
.time
.format
.DateTimeFormatter
;
35 import org
.springframework
.stereotype
.Component
;
38 import java
.util
.regex
.Matcher
;
39 import java
.util
.regex
.Pattern
;
46 @Component("iAPTExcelImport")
47 public class IAPTExcelImport
<CONFIG
extends IAPTImportConfigurator
> extends SimpleExcelTaxonImport
<CONFIG
> {
48 private static final long serialVersionUID
= -747486709409732371L;
49 private static final Logger logger
= Logger
.getLogger(IAPTExcelImport
.class);
50 public static final String ANNOTATION_MARKER_STRING
= "[*]";
53 private static UUID ROOT_UUID
= UUID
.fromString("4137fd2a-20f6-4e70-80b9-f296daf51d82");
55 private static NonViralNameParserImpl nameParser
= NonViralNameParserImpl
.NewInstance();
57 private final static String REGISTRATIONNO_PK
= "RegistrationNo_Pk";
58 private final static String HIGHERTAXON
= "HigherTaxon";
59 private final static String FULLNAME
= "FullName";
60 private final static String AUTHORSSPELLING
= "AuthorsSpelling";
61 private final static String LITSTRING
= "LitString";
62 private final static String REGISTRATION
= "Registration";
63 private final static String TYPE
= "Type";
64 private final static String CAVEATS
= "Caveats";
65 private final static String FULLBASIONYM
= "FullBasionym";
66 private final static String FULLSYNSUBST
= "FullSynSubst";
67 private final static String NOTESTXT
= "NotesTxt";
68 private final static String REGDATE
= "RegDate";
69 private final static String NAMESTRING
= "NameString";
70 private final static String BASIONYMSTRING
= "BasionymString";
71 private final static String SYNSUBSTSTR
= "SynSubstStr";
72 private final static String AUTHORSTRING
= "AuthorString";
74 private static List
<String
> expectedKeys
= Arrays
.asList(new String
[]{
75 REGISTRATIONNO_PK
, HIGHERTAXON
, FULLNAME
, AUTHORSSPELLING
, LITSTRING
, REGISTRATION
, TYPE
, CAVEATS
, FULLBASIONYM
, FULLSYNSUBST
, NOTESTXT
, REGDATE
, NAMESTRING
, BASIONYMSTRING
, SYNSUBSTSTR
, AUTHORSTRING
});
77 private static final Pattern nomRefTokenizeP
= Pattern
.compile("^(.*):\\s([^\\.:]+)\\.(.*?)\\.?$");
78 private static final Pattern
[] datePatterns
= new Pattern
[]{
80 // The order of the patterns is extremely important!!!
82 // all patterns cover the years 1700 - 1999
83 Pattern
.compile("^(?<year>1[7,8,9][0-9]{2})$"), // only year, like '1969'
84 Pattern
.compile("^(?<monthName>\\p{L}+\\.?)\\s(?<day>[0-9]{1,2})(?:st|rd|th)?\\.?,?\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like April 12, 1969 or april 12th 1999
85 Pattern
.compile("^(?<monthName>\\p{L}+\\.?),?\\s?(?<year>(?:1[7,8,9])?[0-9]{2})$"), // April 99 or April, 1999 or Apr. 12
86 Pattern
.compile("^(?<day>[0-9]{1,2})([\\.\\-/])(\\s?)(?<month>[0-1]?[0-9])\\2\\3(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like 12.04.1969 or 12. 04. 1969 or 12/04/1969 or 12-04-1969
87 Pattern
.compile("^(?<day>[0-9]{1,2})([\\.\\-/])(?<month>[IVX]{1,2})\\2(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like 12-VI-1969
88 Pattern
.compile("^(?:(?<day>[0-9]{1,2})(?:\\sde)\\s)(?<monthName>\\p{L}+)\\sde\\s(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full and partial date like 12 de Enero de 1999 or Enero de 1999
89 Pattern
.compile("^(?<month>[0-1]?[0-9])([\\.\\-/])(?<year>(?:1[7,8,9])?[0-9]{2})$"), // partial date like 04.1969 or 04/1969 or 04-1969
90 Pattern
.compile("^(?<year>(?:1[7,8,9])?[0-9]{2})([\\.\\-/])(?<month>[0-1]?[0-9])$"),// partial date like 1999-04
91 Pattern
.compile("^(?<month>[IVX]{1,2})([\\.\\-/])(?<year>(?:1[7,8,9])?[0-9]{2})$"), // partial date like VI-1969
92 Pattern
.compile("^(?<day>[0-9]{1,2})(?:[\\./]|th|rd|st)?\\s(?<monthName>\\p{L}+\\.?),?\\s?(?<year>(?:1[7,8,9])?[0-9]{2})$"), // full date like 12. April 1969 or april 1999 or 22 Dec.1999
94 private static final Pattern typeSplitPattern
= Pattern
.compile("^(?:\"*[Tt]ype: (?<fieldUnit>.*?))(?:[Hh]olotype:(?<holotype>.*?)\\.?)?(?:[Ii]sotype[^:]*:(?<isotype>.*)\\.?)?\\.?$");
96 private static final Pattern collectorPattern
= Pattern
.compile(".*?\\(leg\\.\\s+([^\\)]*)\\)|.*?\\sleg\\.\\s+(.*?)\\.?$");
97 private static final Pattern collectionDataPattern
= Pattern
.compile("^(?<collector>[^,]*),\\s?(?<detail>.*?)\\.?$");
98 private static final Pattern collectorsNumber
= Pattern
.compile("^([nN]o\\.\\s.*)$");
100 // AccessionNumbers: , #.*, n°:?, 96/3293, No..*, -?\w{1,3}-[0-9\-/]*
101 private static final Pattern accessionNumberOnlyPattern
= Pattern
.compile("^(?<accNumber>(?:n°\\:?\\s?|#|No\\.?\\s?)?[\\d\\w\\-/]*)$");
103 private static final Pattern
[] specimenTypePatterns
= new Pattern
[]{
104 Pattern
.compile("^(?<colCode>[A-Z]+|CPC Micropaleontology Lab\\.?)\\s+(?:\\((?<institute>.*[^\\)])\\))(?<accNumber>.*)?$"), // like: GAUF (Gansu Agricultural University) No. 1207-1222
105 Pattern
.compile("^(?<colCode>[A-Z]+|CPC Micropaleontology Lab\\.?)\\s+(?:Coll\\.\\s(?<subCollection>[^\\.,;]*)(.))(?<accNumber>.*)?$"), // like KASSEL Coll. Krasske, Praep. DII 78
106 Pattern
.compile("^(?:Coll\\.\\s(?<subCollection>[^\\.,;]*)(.))(?<institute>.*?)(?<accNumber>Praep\\..*)?$"), // like Coll. Lange-Bertalot, Bot. Inst., Univ. Frankfurt/Main, Germany Praep. Neukaledonien OTL 62
107 Pattern
.compile("^(?<colCode>[A-Z]+)(?:\\s+(?<accNumber>.*))?$"), // identifies the Collection code and takes the rest as accessionNumber if any
110 private static Map
<String
, Integer
> monthFromNameMap
= new HashMap
<>();
113 String
[] ck
= new String
[]{"leden", "únor", "březen", "duben", "květen", "červen", "červenec ", "srpen", "září", "říjen", "listopad", "prosinec"};
114 String
[] fr
= new String
[]{"janvier", "février", "mars", "avril", "mai", "juin", "juillet", "août", "septembre", "octobre", "novembre", "décembre"};
115 String
[] de
= new String
[]{"januar", "februar", "märz", "april", "mai", "juni", "juli", "august", "september", "oktober", "november", "dezember"};
116 String
[] en
= new String
[]{"january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"};
117 String
[] it
= new String
[]{"gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio", "agosto", "settembre", "ottobre", "novembre", "dicembre"};
118 String
[] sp
= new String
[]{"enero", "febrero", "marzo", "abril", "mayo", "junio", "julio", "agosto", "septiembre", "octubre", "noviembre", "diciembre"};
119 String
[] de_abbrev
= new String
[]{"jan.", "feb.", "märz", "apr.", "mai", "jun.", "jul.", "aug.", "sept.", "okt.", "nov.", "dez."};
120 String
[] en_abbrev
= new String
[]{"jan.", "feb.", "mar.", "apr.", "may", "jun.", "jul.", "aug.", "sep.", "oct.", "nov.", "dec."};
121 String
[] port
= new String
[]{"Janeiro", "Fevereiro", "Março", "Abril", "Maio", "Junho", "Julho", "Agosto", "Setembro", "Outubro", "Novembro", "Dezembro"};
122 String
[] rom_num
= new String
[]{"i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix", "x", "xi", "xii"};
124 String
[][] perLang
= new String
[][]{ck
, de
, fr
, en
, it
, sp
, port
, de_abbrev
, en_abbrev
, rom_num
};
126 for (String
[] months
: perLang
) {
127 for(int m
= 1; m
< 13; m
++){
128 monthFromNameMap
.put(months
[m
- 1].toLowerCase(), m
);
133 monthFromNameMap
.put("mar", 3);
134 monthFromNameMap
.put("dec", 12);
135 monthFromNameMap
.put("Februari", 2);
139 DateTimeFormatter formatterYear
= DateTimeFormat
.forPattern("yyyy");
141 private Map
<String
, Collection
> collectionMap
= new HashMap
<>();
145 fieldUnit
, holotype
, isotype
;
147 public SpecimenTypeDesignationStatus
status(){
150 return SpecimenTypeDesignationStatus
.HOLOTYPE();
152 return SpecimenTypeDesignationStatus
.ISOTYPE();
159 private MarkerType markerTypeFossil
= null;
160 private Rank rankUnrankedSupraGeneric
= null;
161 private Rank familyIncertisSedis
= null;
162 private AnnotationType annotationTypeCaveats
= null;
164 private Taxon
makeTaxon(HashMap
<String
, String
> record
, SimpleExcelTaxonImportState
<CONFIG
> state
,
165 TaxonNode higherTaxonNode
, boolean isFossil
) {
167 String line
= state
.getCurrentLine() + ": ";
169 String regNumber
= getValue(record
, REGISTRATIONNO_PK
, false);
170 String regStr
= getValue(record
, REGISTRATION
, true);
171 String titleCacheStr
= getValue(record
, FULLNAME
, true);
172 String nameStr
= getValue(record
, NAMESTRING
, true);
173 String authorStr
= getValue(record
, AUTHORSTRING
, true);
174 String nomRefStr
= getValue(record
, LITSTRING
, true);
175 String authorsSpelling
= getValue(record
, AUTHORSSPELLING
, true);
176 String notesTxt
= getValue(record
, NOTESTXT
, true);
177 String caveats
= getValue(record
, CAVEATS
, true);
178 String fullSynSubstStr
= getValue(record
, FULLSYNSUBST
, true);
179 String synSubstStr
= getValue(record
, SYNSUBSTSTR
, true);
180 String typeStr
= getValue(record
, TYPE
, true);
183 String nomRefTitle
= null;
185 String nomRefPupDate
= null;
186 Partial pupDate
= null;
188 // preprocess nomRef: separate citation, reference detail, publishing date
189 if(!StringUtils
.isEmpty(nomRefStr
)){
190 nomRefStr
= nomRefStr
.trim();
191 Matcher m
= nomRefTokenizeP
.matcher(nomRefStr
);
193 nomRefTitle
= m
.group(1);
194 nomRefDetail
= m
.group(2);
195 nomRefPupDate
= m
.group(3).trim();
197 pupDate
= parseDate(regNumber
, nomRefPupDate
);
198 if (pupDate
!= null) {
199 nomRefTitle
= nomRefTitle
+ ": " + nomRefDetail
+ ". " + pupDate
.toString(formatterYear
) + ".";
201 logger
.warn(csvReportLine(regNumber
, "Pub date", nomRefPupDate
, "in", nomRefStr
, "not parsable"));
204 nomRefTitle
= nomRefStr
;
208 BotanicalName taxonName
= makeBotanicalName(state
, regNumber
, titleCacheStr
, nameStr
, authorStr
, nomRefTitle
);
210 // always add the original strings of parsed data as annotation
211 taxonName
.addAnnotation(Annotation
.NewInstance("imported and parsed data strings:" +
212 "\n - '" + LITSTRING
+ "': "+ nomRefStr
+
213 "\n - '" + TYPE
+ "': " + typeStr
+
214 "\n - '" + REGISTRATION
+ "': " + regStr
215 , AnnotationType
.TECHNICAL(), Language
.DEFAULT()));
217 if(pupDate
!= null) {
218 taxonName
.getNomenclaturalReference().setDatePublished(TimePeriod
.NewInstance(pupDate
));
221 if(!StringUtils
.isEmpty(notesTxt
)){
222 notesTxt
= notesTxt
.replace("Notes: ", "").trim();
223 taxonName
.addAnnotation(Annotation
.NewInstance(notesTxt
, AnnotationType
.EDITORIAL(), Language
.DEFAULT()));
225 if(!StringUtils
.isEmpty(caveats
)){
226 caveats
= caveats
.replace("Caveats: ", "").trim();
227 taxonName
.addAnnotation(Annotation
.NewInstance(caveats
, annotationTypeCaveats(), Language
.DEFAULT()));
232 if(!StringUtils
.isEmpty(authorsSpelling
)){
233 authorsSpelling
= authorsSpelling
.replaceFirst("Author's spelling:", "").replaceAll("\"", "").trim();
235 String
[] authorSpellingTokens
= StringUtils
.split(authorsSpelling
, " ");
236 String
[] nameStrTokens
= StringUtils
.split(nameStr
, " ");
238 ArrayUtils
.reverse(authorSpellingTokens
);
239 ArrayUtils
.reverse(nameStrTokens
);
241 for (int i
= 0; i
< nameStrTokens
.length
; i
++){
242 if(i
< authorSpellingTokens
.length
){
243 nameStrTokens
[i
] = authorSpellingTokens
[i
];
246 ArrayUtils
.reverse(nameStrTokens
);
248 String misspelledNameStr
= StringUtils
.join (nameStrTokens
, ' ');
249 // build the fullnameString of the misspelled name
250 misspelledNameStr
= taxonName
.getTitleCache().replace(nameStr
, misspelledNameStr
);
252 TaxonNameBase misspelledName
= (BotanicalName
) nameParser
.parseReferencedName(misspelledNameStr
, NomenclaturalCode
.ICNAFP
, null);
253 misspelledName
.addRelationshipToName(taxonName
, NameRelationshipType
.MISSPELLING(), null);
254 getNameService().save(misspelledName
);
258 if(!StringUtils
.isEmpty(fullSynSubstStr
)){
259 fullSynSubstStr
= fullSynSubstStr
.replace("Syn. subst.: ", "");
260 BotanicalName replacedSynonymName
= makeBotanicalName(state
, regNumber
, fullSynSubstStr
, synSubstStr
, null, null);
261 replacedSynonymName
.addReplacedSynonym(taxonName
, null, null, null);
262 getNameService().save(replacedSynonymName
);
265 Reference sec
= state
.getConfig().getSecReference();
266 Taxon taxon
= Taxon
.NewInstance(taxonName
, sec
);
270 taxon
.addMarker(Marker
.NewInstance(markerTypeFossil(), true));
274 if(!StringUtils
.isEmpty(typeStr
)){
275 makeTypeData(typeStr
, taxonName
, regNumber
, state
);
278 getTaxonService().save(taxon
);
279 if(higherTaxonNode
!= null){
280 higherTaxonNode
.addChildTaxon(taxon
, null, null);
281 getTaxonNodeService().save(higherTaxonNode
);
288 private void makeTypeData(String typeStr
, BotanicalName taxonName
, String regNumber
, SimpleExcelTaxonImportState
<CONFIG
> state
) {
290 Matcher m
= typeSplitPattern
.matcher(typeStr
);
293 String fieldUnitStr
= m
.group(TypesName
.fieldUnit
.name());
294 // boolean isFieldUnit = typeStr.matches(".*([°']|\\d+\\s?m\\s|\\d+\\s?km\\s).*"); // check for location or unit m, km // makes no sense!!!!
295 FieldUnit fieldUnit
= parseFieldUnit(fieldUnitStr
, regNumber
, state
);
296 if(fieldUnit
== null) {
297 // create a field unit with only a titleCache using the fieldUnitStr substring
298 fieldUnit
= FieldUnit
.NewInstance();
299 fieldUnit
.setTitleCache(fieldUnitStr
, true);
300 getOccurrenceService().save(fieldUnit
);
302 getOccurrenceService().save(fieldUnit
);
305 addSpecimenTypes(taxonName
, fieldUnit
, m
.group(TypesName
.holotype
.name()), TypesName
.holotype
, false, regNumber
);
306 addSpecimenTypes(taxonName
, fieldUnit
, m
.group(TypesName
.isotype
.name()), TypesName
.isotype
, true, regNumber
);
309 // create a field unit with only a titleCache using the full typeStr
310 FieldUnit fieldUnit
= FieldUnit
.NewInstance();
311 fieldUnit
.setTitleCache(typeStr
, true);
312 getOccurrenceService().save(fieldUnit
);
313 logger
.warn(csvReportLine(regNumber
, "Type field can not be parsed", typeStr
));
315 getNameService().save(taxonName
);
319 * Currently only parses the collector, fieldNumber and the collection date.
321 * @param fieldUnitStr
324 * @return null if the fieldUnitStr could not be parsed
326 private FieldUnit
parseFieldUnit(String fieldUnitStr
, String regNumber
, SimpleExcelTaxonImportState
<CONFIG
> state
) {
328 FieldUnit fieldUnit
= null;
330 Matcher m1
= collectorPattern
.matcher(fieldUnitStr
);
332 String collectionData
= m1
.group(1); // like (leg. Metzeltin, 30. 9. 1996)
333 if(collectionData
== null){
334 collectionData
= m1
.group(2); // like leg. Metzeltin, 30. 9. 1996
336 if(collectionData
== null){
340 String collectorStr
= null;
341 String detailStr
= null;
343 String fieldNumber
= null;
345 Matcher m2
= collectionDataPattern
.matcher(collectionData
);
347 collectorStr
= m2
.group("collector");
348 detailStr
= m2
.group("detail");
350 // Try to make sense of the detailStr
351 if(detailStr
!= null){
352 detailStr
= detailStr
.trim();
353 // 1. try to parse as date
354 date
= parseDate(regNumber
, detailStr
);
356 // 2. try to parse as number
357 if(collectorsNumber
.matcher(detailStr
).matches()){
358 fieldNumber
= detailStr
;
362 if(date
== null && fieldNumber
== null){
363 // detailed parsing not possible, so need fo fallback
364 collectorStr
= collectionData
;
368 if(collectorStr
!= null) {
369 fieldUnit
= FieldUnit
.NewInstance();
370 GatheringEvent ge
= GatheringEvent
.NewInstance();
372 TeamOrPersonBase agent
= state
.getAgentBase(collectorStr
);
374 agent
= Person
.NewTitledInstance(collectorStr
);
375 getAgentService().save(agent
);
376 state
.putAgentBase(collectorStr
, agent
);
378 ge
.setCollector(agent
);
381 ge
.setGatheringDate(date
);
384 getEventBaseService().save(ge
);
385 fieldUnit
.setGatheringEvent(ge
);
387 if(fieldNumber
!= null) {
388 fieldUnit
.setFieldNumber(fieldNumber
);
390 getOccurrenceService().save(fieldUnit
);
397 private Partial
parseDate(String regNumber
, String dateStr
) {
399 Partial pupDate
= null;
400 boolean parseError
= false;
404 String monthName
= null;
407 for(Pattern p
: datePatterns
){
408 Matcher m2
= p
.matcher(dateStr
);
411 year
= m2
.group("year");
412 } catch (IllegalArgumentException e
){
413 // named capture group not found
416 month
= m2
.group("month");
417 } catch (IllegalArgumentException e
){
418 // named capture group not found
422 monthName
= m2
.group("monthName");
423 month
= monthFromName(monthName
, regNumber
);
427 } catch (IllegalArgumentException e
){
428 // named capture group not found
431 day
= m2
.group("day");
432 } catch (IllegalArgumentException e
){
433 // named capture group not found
437 if (year
.length() == 2) {
438 // it is an abbreviated year from the 19** years
450 List
<DateTimeFieldType
> types
= new ArrayList
<>();
451 List
<Integer
> values
= new ArrayList
<>();
453 types
.add(DateTimeFieldType
.year());
454 values
.add(Integer
.parseInt(year
));
456 types
.add(DateTimeFieldType
.monthOfYear());
457 values
.add(Integer
.parseInt(month
));
460 types
.add(DateTimeFieldType
.dayOfMonth());
461 values
.add(Integer
.parseInt(day
));
463 pupDate
= new Partial(types
.toArray(new DateTimeFieldType
[types
.size()]), ArrayUtils
.toPrimitive(values
.toArray(new Integer
[values
.size()])));
468 private String
monthFromName(String monthName
, String regNumber
) {
470 Integer month
= monthFromNameMap
.get(monthName
.toLowerCase());
472 logger
.warn(csvReportLine(regNumber
, "Unknown month name", monthName
));
475 return month
.toString();
480 private void addSpecimenTypes(BotanicalName taxonName
, FieldUnit fieldUnit
, String typeStr
, TypesName typeName
, boolean multiple
, String regNumber
){
482 if(StringUtils
.isEmpty(typeStr
)){
485 typeStr
= typeStr
.trim().replaceAll("\\.$", "");
487 Collection collection
= null;
488 DerivedUnit specimen
= null;
490 List
<DerivedUnit
> specimens
= new ArrayList
<>();
492 String
[] tokens
= typeStr
.split("\\s?,\\s?");
493 for (String t
: tokens
) {
494 // command to list all complex parsabel types:
495 // csvcut -t -c RegistrationNo_Pk,Type iapt.csv | csvgrep -c Type -m "Holotype" | egrep -o 'Holotype:\s([A-Z]*\s)[^.]*?'
496 // csvcut -t -c RegistrationNo_Pk,Type iapt.csv | csvgrep -c Type -m "Holotype" | egrep -o 'Isotype[^:]*:\s([A-Z]*\s)[^.]*?'
499 // trying to parse the string
500 specimen
= parseSpecimenType(fieldUnit
, typeName
, collection
, t
, regNumber
);
501 if(specimen
!= null){
502 specimens
.add(specimen
);
504 // parsing was not successful make simple specimen
505 specimens
.add(makeSpecimenType(fieldUnit
, t
));
510 specimen
= parseSpecimenType(fieldUnit
, typeName
, collection
, typeStr
, regNumber
);
511 if(specimen
!= null) {
512 specimens
.add(specimen
);
513 // remember current collection
514 collection
= specimen
.getCollection();
516 // parsing was not successful make simple specimen
517 specimens
.add(makeSpecimenType(fieldUnit
, typeStr
));
521 for(DerivedUnit s
: specimens
){
522 taxonName
.addSpecimenTypeDesignation(s
, typeName
.status(), null, null, null, false, true);
526 private DerivedUnit
makeSpecimenType(FieldUnit fieldUnit
, String titleCache
) {
527 DerivedUnit specimen
;DerivedUnitFacade facade
= DerivedUnitFacade
.NewInstance(SpecimenOrObservationType
.PreservedSpecimen
, fieldUnit
);
528 facade
.setTitleCache(titleCache
.trim(), true);
529 specimen
= facade
.innerDerivedUnit();
542 private DerivedUnit
parseSpecimenType(FieldUnit fieldUnit
, TypesName typeName
, Collection collection
, String text
, String regNumber
) {
544 DerivedUnit specimen
= null;
546 String collectionCode
= null;
547 String subCollectionStr
= null;
548 String instituteStr
= null;
549 String accessionNumber
= null;
551 boolean unusualAccessionNumber
= false;
555 // 1. For Isotypes often the accession number is noted alone if the
556 // preceeding entry has a collection code.
557 if(typeName
.equals(TypesName
.isotype
) && collection
!= null){
558 Matcher m
= accessionNumberOnlyPattern
.matcher(text
);
561 accessionNumber
= m
.group("accNumber");
562 specimen
= makeSpecimenType(fieldUnit
, collection
, accessionNumber
);
563 } catch (IllegalArgumentException e
){
564 // match group acc_number not found
569 //2. try it the 'normal' way
570 if(specimen
== null) {
571 for (Pattern p
: specimenTypePatterns
) {
572 Matcher m
= p
.matcher(text
);
574 // collection code is mandatory
576 collectionCode
= m
.group("colCode");
577 } catch (IllegalArgumentException e
){
578 // match group colCode not found
581 subCollectionStr
= m
.group("subCollection");
582 } catch (IllegalArgumentException e
){
583 // match group subCollection not found
586 instituteStr
= m
.group("institute");
587 } catch (IllegalArgumentException e
){
588 // match group col_name not found
591 accessionNumber
= m
.group("accNumber");
593 // try to improve the accessionNumber
594 if(accessionNumber
!= null) {
595 accessionNumber
= accessionNumber
.trim();
596 Matcher m2
= accessionNumberOnlyPattern
.matcher(accessionNumber
);
597 String betterAccessionNumber
= null;
600 betterAccessionNumber
= m
.group("accNumber");
601 } catch (IllegalArgumentException e
) {
602 // match group acc_number not found
605 if (betterAccessionNumber
!= null) {
606 accessionNumber
= betterAccessionNumber
;
608 unusualAccessionNumber
= true;
612 } catch (IllegalArgumentException e
){
613 // match group acc_number not found
616 if(collectionCode
== null && instituteStr
== null){
617 logger
.warn(csvReportLine(regNumber
, "neither 'collectionCode' nor 'institute' found in ", text
));
620 collection
= getCollection(collectionCode
, instituteStr
, subCollectionStr
);
621 specimen
= makeSpecimenType(fieldUnit
, collection
, accessionNumber
);
626 if(specimen
== null) {
627 logger
.warn(csvReportLine(regNumber
, "Could not parse specimen fieldUnit", typeName
.name().toString(), text
));
629 if(unusualAccessionNumber
){
630 logger
.warn(csvReportLine(regNumber
, "Unusual accession number", typeName
.name().toString(), text
, accessionNumber
));
635 private DerivedUnit
makeSpecimenType(FieldUnit fieldUnit
, Collection collection
, String accessionNumber
) {
637 DerivedUnitFacade facade
= DerivedUnitFacade
.NewInstance(SpecimenOrObservationType
.PreservedSpecimen
, fieldUnit
);
638 facade
.setCollection(collection
);
639 if(accessionNumber
!= null){
640 facade
.setAccessionNumber(accessionNumber
);
642 return facade
.innerDerivedUnit();
645 private BotanicalName
makeBotanicalName(SimpleExcelTaxonImportState
<CONFIG
> state
, String regNumber
, String titleCacheStr
, String nameStr
,
646 String authorStr
, String nomRefTitle
) {
648 BotanicalName taxonName
;// cache field for the taxonName.titleCache
649 String taxonNameTitleCache
= null;
650 Map
<String
, AnnotationType
> nameAnnotations
= new HashMap
<>();
652 // TitleCache preprocessing
653 if(titleCacheStr
.endsWith(ANNOTATION_MARKER_STRING
) || (authorStr
!= null && authorStr
.endsWith(ANNOTATION_MARKER_STRING
))){
654 nameAnnotations
.put("Author abbreviation not checked.", AnnotationType
.EDITORIAL());
655 titleCacheStr
= titleCacheStr
.replace(ANNOTATION_MARKER_STRING
, "").trim();
656 authorStr
= authorStr
.replace(ANNOTATION_MARKER_STRING
, "").trim();
659 // parse the full taxon name
660 if(!StringUtils
.isEmpty(nomRefTitle
)){
661 String referenceSeparator
= nomRefTitle
.startsWith("in ") ?
" " : ", ";
662 String taxonFullNameStr
= titleCacheStr
+ referenceSeparator
+ nomRefTitle
;
663 logger
.debug(":::::" + taxonFullNameStr
);
664 taxonName
= (BotanicalName
) nameParser
.parseReferencedName(taxonFullNameStr
, NomenclaturalCode
.ICNAFP
, null);
666 taxonName
= (BotanicalName
) nameParser
.parseFullName(titleCacheStr
, NomenclaturalCode
.ICNAFP
, null);
669 taxonNameTitleCache
= taxonName
.getTitleCache().trim();
670 if (taxonName
.isProtectedTitleCache()) {
671 logger
.warn(csvReportLine(regNumber
, "Name could not be parsed", titleCacheStr
));
674 boolean doRestoreTitleCacheStr
= false;
676 // Check if titleCache and nameCache are plausible
677 String titleCacheCompareStr
= titleCacheStr
;
678 String nameCache
= taxonName
.getNameCache();
679 String nameCompareStr
= nameStr
;
680 if(taxonName
.isBinomHybrid()){
681 titleCacheCompareStr
= titleCacheCompareStr
.replace(" x ", " ×");
682 nameCompareStr
= nameCompareStr
.replace(" x ", " ×");
684 if(taxonName
.isMonomHybrid()){
685 titleCacheCompareStr
= titleCacheCompareStr
.replaceAll("^X ", "× ");
686 nameCompareStr
= nameCompareStr
.replace("^X ", "× ");
688 if(authorStr
!= null && authorStr
.contains(" et ")){
689 titleCacheCompareStr
= titleCacheCompareStr
.replaceAll(" et ", " & ");
691 if (!taxonNameTitleCache
.equals(titleCacheCompareStr
)) {
692 logger
.warn(csvReportLine(regNumber
, "The generated titleCache differs from the imported string", taxonNameTitleCache
, " != ", titleCacheStr
, " ==> original titleCacheStr has been restored"));
693 doRestoreTitleCacheStr
= true;
695 if (!nameCache
.trim().equals(nameCompareStr
)) {
696 logger
.warn(csvReportLine(regNumber
, "The parsed nameCache differs from field '" + NAMESTRING
+ "'", nameCache
, " != ", nameCompareStr
));
700 //nameParser.handleAuthors(taxonName, titleCacheStr, authorStr);
701 //if (!titleCacheStr.equals(taxonName.getTitleCache())) {
702 // logger.warn(regNumber + ": titleCache has changed after setting authors, will restore original titleCacheStr");
703 // doRestoreTitleCacheStr = true;
706 if(doRestoreTitleCacheStr
){
707 taxonName
.setTitleCache(titleCacheStr
, true);
711 replaceAuthorNamesAndNomRef(state
, taxonName
);
715 if(!nameAnnotations
.isEmpty()){
716 for(String text
: nameAnnotations
.keySet()){
717 taxonName
.addAnnotation(Annotation
.NewInstance(text
, nameAnnotations
.get(text
), Language
.DEFAULT()));
719 getNameService().save(taxonName
);
728 private TaxonNode
getClassificationRootNode(IAPTImportState state
) {
730 // Classification classification = state.getClassification();
731 // if (classification == null){
732 // IAPTImportConfigurator config = state.getConfig();
733 // classification = Classification.NewInstance(state.getConfig().getClassificationName());
734 // classification.setUuid(config.getClassificationUuid());
735 // classification.setReference(config.getSecReference());
736 // classification = getClassificationService().find(state.getConfig().getClassificationUuid());
738 TaxonNode rootNode
= state
.getRootNode();
739 if (rootNode
== null){
740 rootNode
= getTaxonNodeService().find(ROOT_UUID
);
742 if (rootNode
== null){
743 Classification classification
= state
.getClassification();
744 if (classification
== null){
745 Reference sec
= state
.getSecReference();
746 String classificationName
= state
.getConfig().getClassificationName();
747 Language language
= Language
.DEFAULT();
748 classification
= Classification
.NewInstance(classificationName
, sec
, language
);
749 state
.setClassification(classification
);
750 classification
.setUuid(state
.getConfig().getClassificationUuid());
751 classification
.getRootNode().setUuid(ROOT_UUID
);
752 getClassificationService().save(classification
);
754 rootNode
= classification
.getRootNode();
755 state
.setRootNode(rootNode
);
760 private Collection
getCollection(String collectionCode
, String instituteStr
, String subCollectionStr
){
762 Collection superCollection
= null;
763 if(subCollectionStr
!= null){
764 superCollection
= getCollection(collectionCode
, instituteStr
, null);
765 collectionCode
= subCollectionStr
;
769 final String key
= collectionCode
+ "-#i:" + StringUtils
.defaultString(instituteStr
);
771 Collection collection
= collectionMap
.get(key
);
773 if(collection
== null) {
774 collection
= Collection
.NewInstance();
775 collection
.setCode(collectionCode
);
776 if(instituteStr
!= null){
777 collection
.setInstitute(Institution
.NewNamedInstance(instituteStr
));
779 if(superCollection
!= null){
780 collection
.setSuperCollection(superCollection
);
782 collectionMap
.put(key
, collection
);
783 getCollectionService().save(collection
);
793 * @param doUnescapeHtmlEntities
796 private String
getValue(HashMap
<String
, String
> record
, String originalKey
, boolean doUnescapeHtmlEntities
) {
797 String value
= record
.get(originalKey
);
799 value
= fixCharacters(value
);
801 if (! StringUtils
.isBlank(value
)) {
802 if (logger
.isDebugEnabled()) {
803 logger
.debug(originalKey
+ ": " + value
);
805 value
= CdmUtils
.removeDuplicateWhitespace(value
.trim()).toString();
806 if(doUnescapeHtmlEntities
){
807 value
= StringEscapeUtils
.unescapeHtml(value
);
816 * Fixes broken characters.
818 * http://dev.e-taxonomy.eu/redmine/issues/6035
823 private String
fixCharacters(String value
) {
825 value
= StringUtils
.replace(value
, "s$K", "š");
826 value
= StringUtils
.replace(value
, "n$K", "ň");
827 value
= StringUtils
.replace(value
, "e$K", "ě");
828 value
= StringUtils
.replace(value
, "r$K", "ř");
829 value
= StringUtils
.replace(value
, "c$K", "č");
830 value
= StringUtils
.replace(value
, "z$K", "ž");
831 value
= StringUtils
.replace(value
, "S>U$K", "Š");
832 value
= StringUtils
.replace(value
, "C>U$K", "Č");
833 value
= StringUtils
.replace(value
, "R>U$K", "Ř");
834 value
= StringUtils
.replace(value
, "Z>U$K", "Ž");
835 value
= StringUtils
.replace(value
, "g$K", "ǧ");
836 value
= StringUtils
.replace(value
, "s$A", "ś");
837 value
= StringUtils
.replace(value
, "n$A", "ń");
838 value
= StringUtils
.replace(value
, "c$A", "ć");
839 value
= StringUtils
.replace(value
, "e$E", "ę");
840 value
= StringUtils
.replace(value
, "o$H", "õ");
841 value
= StringUtils
.replace(value
, "s$C", "ş");
842 value
= StringUtils
.replace(value
, "t$C", "ț");
843 value
= StringUtils
.replace(value
, "S>U$C", "Ş");
844 value
= StringUtils
.replace(value
, "a$O", "å");
845 value
= StringUtils
.replace(value
, "A>U$O", "Å");
846 value
= StringUtils
.replace(value
, "u$O", "ů");
847 value
= StringUtils
.replace(value
, "g$B", "ğ");
848 value
= StringUtils
.replace(value
, "g$B", "ĕ");
849 value
= StringUtils
.replace(value
, "a$B", "ă");
850 value
= StringUtils
.replace(value
, "l$/", "ł");
851 value
= StringUtils
.replace(value
, ">i", "ı");
852 value
= StringUtils
.replace(value
, "i$U", "ï");
854 value
= StringUtils
.replace(value
, "ý", "ý");
855 value
= StringUtils
.replace(value
, ">L", "Ł"); // corrected rule
856 value
= StringUtils
.replace(value
, "E>U$D", "З");
857 value
= StringUtils
.replace(value
, "S>U$E", "Ş");
858 value
= StringUtils
.replace(value
, "s$E", "ş");
860 value
= StringUtils
.replace(value
, "c$k", "č");
861 value
= StringUtils
.replace(value
, " U$K", " Š");
868 * Stores taxa records in DB
871 protected void firstPass(SimpleExcelTaxonImportState
<CONFIG
> state
) {
873 String lineNumber
= "L#" + state
.getCurrentLine() + ": ";
874 logger
.setLevel(Level
.DEBUG
);
875 HashMap
<String
, String
> record
= state
.getOriginalRecord();
876 logger
.debug(lineNumber
+ record
.toString());
878 Set
<String
> keys
= record
.keySet();
879 for (String key
: keys
) {
880 if (! expectedKeys
.contains(key
)){
881 logger
.warn(lineNumber
+ "Unexpected Key: " + key
);
885 String reg_id
= record
.get(REGISTRATIONNO_PK
);
888 String higherTaxaString
= record
.get(HIGHERTAXON
);
889 boolean isFossil
= false;
890 if(higherTaxaString
.startsWith("FOSSIL ")){
891 higherTaxaString
= higherTaxaString
.replace("FOSSIL ", "");
894 TaxonNode higherTaxon
= getHigherTaxon(higherTaxaString
, (IAPTImportState
)state
);
897 Taxon taxon
= makeTaxon(record
, state
, higherTaxon
, isFossil
);
899 logger
.warn(lineNumber
+ "taxon could not be created and is null");
902 ((IAPTImportState
)state
).setCurrentTaxon(taxon
);
908 private TaxonNode
getHigherTaxon(String higherTaxaString
, IAPTImportState state
) {
909 String
[] higherTaxaNames
= higherTaxaString
.toLowerCase().replaceAll("[\\[\\]]", "").split(":");
910 TaxonNode higherTaxonNode
= null;
912 ITaxonTreeNode rootNode
= getClassificationRootNode(state
);
913 for (String htn
: higherTaxaNames
) {
914 htn
= StringUtils
.capitalize(htn
.trim());
915 Taxon higherTaxon
= state
.getHigherTaxon(htn
);
916 if (higherTaxon
!= null){
917 higherTaxonNode
= higherTaxon
.getTaxonNodes().iterator().next();
919 BotanicalName name
= makeHigherTaxonName(state
, htn
);
920 Reference sec
= state
.getSecReference();
921 higherTaxon
= Taxon
.NewInstance(name
, sec
);
922 getTaxonService().save(higherTaxon
);
923 higherTaxonNode
= rootNode
.addChildTaxon(higherTaxon
, sec
, null);
924 state
.putHigherTaxon(htn
, higherTaxon
);
925 getClassificationService().saveTreeNode(higherTaxonNode
);
927 rootNode
= higherTaxonNode
;
929 return higherTaxonNode
;
932 private BotanicalName
makeHigherTaxonName(IAPTImportState state
, String name
) {
934 Rank rank
= guessRank(name
);
936 BotanicalName taxonName
= BotanicalName
.NewInstance(rank
);
937 taxonName
.addSource(makeOriginalSource(state
));
938 taxonName
.setGenusOrUninomial(StringUtils
.capitalize(name
));
942 private Rank
guessRank(String name
) {
945 name
= name
.replaceAll("\\(.*\\)", "").trim();
947 if(name
.matches("^Plantae$|^Fungi$")){
948 return Rank
.KINGDOM();
949 } else if(name
.matches("^Incertae sedis$|^No group assigned$")){
950 return rankFamilyIncertisSedis();
951 } else if(name
.matches(".*phyta$|.*mycota$")){
952 return Rank
.SECTION_BOTANY();
953 } else if(name
.matches(".*phytina$|.*mycotina$")){
954 return Rank
.SUBSECTION_BOTANY();
955 } else if(name
.matches("Gymnospermae$|.*ones$")){ // Monocotyledones, Dicotyledones
956 return rankUnrankedSupraGeneric();
957 } else if(name
.matches(".*opsida$|.*phyceae$|.*mycetes$|.*ones$|^Musci$|^Hepaticae$")){
959 } else if(name
.matches(".*idae$|.*phycidae$|.*mycetidae$")){
960 return Rank
.SUBCLASS();
961 } else if(name
.matches(".*ales$")){
963 } else if(name
.matches(".*ineae$")){
964 return Rank
.SUBORDER();
965 } else if(name
.matches(".*aceae$")){
966 return Rank
.FAMILY();
967 } else if(name
.matches(".*oideae$")){
968 return Rank
.SUBFAMILY();
970 // if(name.matches(".*eae$")){
971 // return Rank.TRIBE();
973 if(name
.matches(".*inae$")){
974 return Rank
.SUBTRIBE();
975 } else if(name
.matches(".*ae$")){
976 return Rank
.FAMILY();
978 return Rank
.UNKNOWN_RANK();
981 private Rank
rankUnrankedSupraGeneric() {
983 if(rankUnrankedSupraGeneric
== null){
984 rankUnrankedSupraGeneric
= Rank
.NewInstance(RankClass
.Suprageneric
, "Unranked supra generic", " ", " ");
985 getTermService().save(rankUnrankedSupraGeneric
);
987 return rankUnrankedSupraGeneric
;
990 private Rank
rankFamilyIncertisSedis() {
992 if(familyIncertisSedis
== null){
993 familyIncertisSedis
= Rank
.NewInstance(RankClass
.Suprageneric
, "Family incertis sedis", " ", " ");
994 getTermService().save(familyIncertisSedis
);
996 return familyIncertisSedis
;
999 private AnnotationType
annotationTypeCaveats(){
1000 if(annotationTypeCaveats
== null){
1001 annotationTypeCaveats
= AnnotationType
.NewInstance("Caveats", "Caveats", "");
1002 getTermService().save(annotationTypeCaveats
);
1004 return annotationTypeCaveats
;
1012 private IdentifiableSource
makeOriginalSource(IAPTImportState state
) {
1013 return IdentifiableSource
.NewDataImportInstance("line: " + state
.getCurrentLine(), null, state
.getConfig().getSourceReference());
1017 private Reference
makeReference(IAPTImportState state
, UUID uuidRef
) {
1018 Reference ref
= state
.getReference(uuidRef
);
1020 ref
= getReferenceService().find(uuidRef
);
1021 state
.putReference(uuidRef
, ref
);
1026 private MarkerType
markerTypeFossil(){
1027 if(this.markerTypeFossil
== null){
1028 markerTypeFossil
= MarkerType
.NewInstance("isFossilTaxon", "isFossil", null);
1029 getTermService().save(this.markerTypeFossil
);
1031 return markerTypeFossil
;
1034 private String
csvReportLine(String regId
, String message
, String
... fields
){
1035 StringBuilder out
= new StringBuilder("regID#");
1036 out
.append(regId
).append(",\"").append(message
).append('"');
1038 for(String f
: fields
){
1039 out
.append(",\"").append(f
).append('"');
1041 return out
.toString();