2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.cyprus
;
12 import java
.util
.Arrays
;
13 import java
.util
.HashMap
;
14 import java
.util
.HashSet
;
15 import java
.util
.List
;
17 import java
.util
.UUID
;
19 import org
.apache
.commons
.lang
.StringUtils
;
20 import org
.apache
.log4j
.Logger
;
21 import org
.springframework
.security
.authentication
.ProviderManager
;
22 import org
.springframework
.security
.authentication
.UsernamePasswordAuthenticationToken
;
23 import org
.springframework
.security
.core
.Authentication
;
24 import org
.springframework
.security
.core
.context
.SecurityContextHolder
;
25 import org
.springframework
.stereotype
.Component
;
27 import eu
.etaxonomy
.cdm
.api
.application
.CdmApplicationController
;
28 import eu
.etaxonomy
.cdm
.common
.CdmUtils
;
29 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.IInputTransformer
;
30 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.UndefinedTransformerMethodException
;
31 import eu
.etaxonomy
.cdm
.io
.excel
.common
.ExcelImporterBase
;
32 import eu
.etaxonomy
.cdm
.model
.agent
.Person
;
33 import eu
.etaxonomy
.cdm
.model
.common
.Language
;
34 import eu
.etaxonomy
.cdm
.model
.common
.Marker
;
35 import eu
.etaxonomy
.cdm
.model
.common
.MarkerType
;
36 import eu
.etaxonomy
.cdm
.model
.common
.User
;
37 import eu
.etaxonomy
.cdm
.model
.description
.Distribution
;
38 import eu
.etaxonomy
.cdm
.model
.description
.Feature
;
39 import eu
.etaxonomy
.cdm
.model
.description
.PresenceTerm
;
40 import eu
.etaxonomy
.cdm
.model
.description
.TaxonDescription
;
41 import eu
.etaxonomy
.cdm
.model
.description
.TextData
;
42 import eu
.etaxonomy
.cdm
.model
.location
.NamedArea
;
43 import eu
.etaxonomy
.cdm
.model
.location
.TdwgArea
;
44 import eu
.etaxonomy
.cdm
.model
.name
.BotanicalName
;
45 import eu
.etaxonomy
.cdm
.model
.name
.NomenclaturalCode
;
46 import eu
.etaxonomy
.cdm
.model
.name
.NonViralName
;
47 import eu
.etaxonomy
.cdm
.model
.name
.Rank
;
48 import eu
.etaxonomy
.cdm
.model
.reference
.Reference
;
49 import eu
.etaxonomy
.cdm
.model
.taxon
.Classification
;
50 import eu
.etaxonomy
.cdm
.model
.taxon
.Synonym
;
51 import eu
.etaxonomy
.cdm
.model
.taxon
.SynonymRelationshipType
;
52 import eu
.etaxonomy
.cdm
.model
.taxon
.Taxon
;
53 import eu
.etaxonomy
.cdm
.model
.taxon
.TaxonBase
;
54 import eu
.etaxonomy
.cdm
.strategy
.parser
.INonViralNameParser
;
55 import eu
.etaxonomy
.cdm
.strategy
.parser
.NonViralNameParserImpl
;
58 * @author a.babadshanjan
64 public class CyprusExcelImport
extends ExcelImporterBase
<CyprusImportState
> {
65 private static final Logger logger
= Logger
.getLogger(CyprusExcelImport
.class);
67 public static Set
<String
> validMarkers
= new HashSet
<String
>(Arrays
.asList(new String
[]{"", "valid", "accepted", "a", "v", "t"}));
68 public static Set
<String
> synonymMarkers
= new HashSet
<String
>(Arrays
.asList(new String
[]{"", "invalid", "synonym", "s", "i"}));
72 protected boolean isIgnore(CyprusImportState state
) {
78 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
81 protected boolean doCheck(CyprusImportState state
) {
82 logger
.warn("DoCheck not yet implemented for CyprusExcelImport");
86 // protected static final String ID_COLUMN = "Id";
87 protected static final String SPECIES_COLUMN
= "species";
88 protected static final String SUBSPECIES_COLUMN
= "subspecies";
89 protected static final String GENUS_COLUMN
= "genus";
90 protected static final String FAMILY_COLUMN
= "family";
91 protected static final String DIVISION_COLUMN
= "division";
92 protected static final String HOMOTYPIC_SYNONYM_COLUMN
= "homotypic synonyms";
93 protected static final String HETEROTYPIC_SYNONYMS_COLUMN
= "heterotypic synonyms";
94 protected static final String ENDEMISM_COLUMN
= "endemism";
96 protected static final String STATUS_COLUMN
= "status";
97 protected static final String RED_DATA_BOOK_CATEGORY_COLUMN
= "red data book category";
98 protected static final String SYSTEMATICS_COLUMN
= "systematics";
102 // TODO: This enum is for future use (perhaps).
103 protected enum Columns
{
106 Subspecies("subspecies"),
109 Division("division"),
110 HomotypicSynonyms("homotypic synonyms"),
111 HeterotypicSynonyms("heterotypic synonyms"),
113 Endemism("endemism");
116 private String value
;
118 private Columns(String head
) {
122 public String
head() {
126 public String
value() {
133 protected boolean analyzeRecord(HashMap
<String
, String
> record
, CyprusImportState state
) {
135 boolean success
= true;
136 Set
<String
> keys
= record
.keySet();
138 CyprusRow cyprusRow
= new CyprusRow();
139 state
.setCyprusRow(cyprusRow
);
141 for (String originalKey
: keys
) {
143 String indexedKey
= CdmUtils
.removeDuplicateWhitespace(originalKey
.trim()).toString();
144 String
[] split
= indexedKey
.split("_");
145 String key
= split
[0];
146 if (split
.length
> 1){
147 String indexString
= split
[1];
149 index
= Integer
.valueOf(indexString
);
150 } catch (NumberFormatException e
) {
151 String message
= "Index must be integer";
152 logger
.error(message
);
157 String value
= (String
) record
.get(indexedKey
);
158 if (! StringUtils
.isBlank(value
)) {
159 if (logger
.isDebugEnabled()) { logger
.debug(key
+ ": " + value
); }
160 value
= CdmUtils
.removeDuplicateWhitespace(value
.trim()).toString();
166 if (key
.equalsIgnoreCase(SPECIES_COLUMN
)) {
167 // int ivalue = floatString2IntValue(value);
168 cyprusRow
.setSpecies(value
);
170 } else if(key
.equalsIgnoreCase(SUBSPECIES_COLUMN
)) {
171 cyprusRow
.setSubspecies(value
);
173 } else if(key
.equalsIgnoreCase(HOMOTYPIC_SYNONYM_COLUMN
)) {
174 cyprusRow
.setHomotypicSynonyms(value
);
176 } else if(key
.equalsIgnoreCase(HETEROTYPIC_SYNONYMS_COLUMN
)) {
177 cyprusRow
.setHeterotypicSynonyms(value
);
179 } else if(key
.equalsIgnoreCase(ENDEMISM_COLUMN
)) {
180 cyprusRow
.setEndemism(value
);
182 } else if(key
.equalsIgnoreCase(STATUS_COLUMN
)) {
183 cyprusRow
.setStatus(value
);
185 } else if(key
.equalsIgnoreCase(RED_DATA_BOOK_CATEGORY_COLUMN
)) {
186 cyprusRow
.setRedDataBookCategory(value
);
188 } else if(key
.equalsIgnoreCase(SYSTEMATICS_COLUMN
)) {
189 cyprusRow
.setSystematics(value
);
191 } else if(key
.equalsIgnoreCase(GENUS_COLUMN
)) {
192 cyprusRow
.setGenus(value
);
194 } else if(key
.equalsIgnoreCase(FAMILY_COLUMN
)) {
195 cyprusRow
.setFamily(value
);
197 } else if(key
.equalsIgnoreCase(DIVISION_COLUMN
)) {
198 cyprusRow
.setDivision(value
);
202 logger
.error("Unexpected column header " + key
);
208 private static INonViralNameParser nameParser
= NonViralNameParserImpl
.NewInstance();
209 private static NomenclaturalCode nc
= NomenclaturalCode
.ICBN
;
210 private Feature redBookCategory
;
211 private Feature endemism
;
212 private PresenceTerm indigenous
;
213 private PresenceTerm indigenousDoubtful
;
214 private PresenceTerm cultivatedDoubtful
;
216 private PresenceTerm casual
;
217 private PresenceTerm casualDoubtful
;
218 private PresenceTerm nonInvasive
;
219 private PresenceTerm nonInvasiveDoubtful
;
220 private PresenceTerm invasive
;
221 private PresenceTerm invasiveDoubtful
;
222 private PresenceTerm questionable
;
223 private PresenceTerm questionableDoubtful
;
225 private boolean termsCreated
= false;
227 private boolean makeTerms(CyprusImportState state
) {
228 if (termsCreated
== false){
229 IInputTransformer transformer
= state
.getTransformer();
233 UUID redBookUuid
= transformer
.getFeatureUuid("Red book");
234 redBookCategory
= this.getFeature(state
, redBookUuid
, "Red book category", "Red data book category", "Red book");
235 getTermService().save(redBookCategory
);
237 UUID endemismUuid
= transformer
.getFeatureUuid("Endemism");
238 endemism
= this.getFeature(state
, endemismUuid
, "Endemism", "Endemism", "Endemism");
239 getTermService().save(endemism
);
243 UUID indigenousUuid
= transformer
.getPresenceTermUuid("IN");
244 indigenous
= this.getPresenceTerm(state
, indigenousUuid
, "Indigenous", "Indigenous", "IN");
245 getTermService().save(indigenous
);
246 UUID indigenousDoubtfulUuid
= transformer
.getPresenceTermUuid("IN?");
247 indigenousDoubtful
= this.getPresenceTerm(state
, indigenousDoubtfulUuid
, "Indigenous?", "Indigenous?", "IN?");
248 getTermService().save(indigenousDoubtful
);
250 UUID cultivatedDoubtfulUuid
= transformer
.getPresenceTermUuid("CU?");
251 cultivatedDoubtful
= this.getPresenceTerm(state
, cultivatedDoubtfulUuid
, "Cultivated?", "Cultivated?", "CU?");
252 getTermService().save(cultivatedDoubtful
);
255 UUID casualUuid
= transformer
.getPresenceTermUuid("CA");
256 casual
= this.getPresenceTerm(state
, casualUuid
, "Casual", "Casual", "CA");
257 getTermService().save(casual
);
258 UUID casualDoubtfulUuid
= transformer
.getPresenceTermUuid("CA?");
259 casualDoubtful
= this.getPresenceTerm(state
, casualDoubtfulUuid
, "Casual?", "Casual?", "CA?");
260 getTermService().save(casualDoubtful
);
263 UUID nonInvasiveUuid
= transformer
.getPresenceTermUuid("NN");
264 nonInvasive
= this.getPresenceTerm(state
, nonInvasiveUuid
, "Naturalized non-invasive", "Naturalized non-invasive", "NN");
265 getTermService().save(nonInvasive
);
266 UUID nonInvasiveDoubtfulUuid
= transformer
.getPresenceTermUuid("NN?");
267 nonInvasiveDoubtful
= this.getPresenceTerm(state
, nonInvasiveDoubtfulUuid
, "Naturalized non-invasive?", "Naturalized non-invasive?", "NN?");
268 getTermService().save(nonInvasiveDoubtful
);
270 UUID invasiveUuid
= transformer
.getPresenceTermUuid("NA");
271 invasive
= this.getPresenceTerm(state
, invasiveUuid
, "Naturalized invasive", "Naturalized invasive", "NA");
272 getTermService().save(invasive
);
273 UUID invasiveDoubtfulUuid
= transformer
.getPresenceTermUuid("NA?");
274 invasiveDoubtful
= this.getPresenceTerm(state
, invasiveDoubtfulUuid
, "Naturalized invasive?", "Naturalized invasive?", "NA?");
275 getTermService().save(invasiveDoubtful
);
277 UUID questionableUuid
= transformer
.getPresenceTermUuid("Q");
278 questionable
= this.getPresenceTerm(state
, questionableUuid
, "Questionable", "Questionable", "Q");
279 getTermService().save(questionable
);
280 UUID questionableDoubtfulUuid
= transformer
.getPresenceTermUuid("Q?");
281 questionableDoubtful
= this.getPresenceTerm(state
, questionableDoubtfulUuid
, "Questionable?", "Questionable?", "Q?");
282 getTermService().save(questionableDoubtful
);
287 } catch (UndefinedTransformerMethodException e
) {
297 * Stores taxa records in DB
300 protected boolean firstPass(CyprusImportState state
) {
302 boolean success
= true;
304 CyprusRow taxonLight
= state
.getCyprusRow();
305 Reference citation
= null;
306 String microCitation
= null;
309 String speciesStr
= taxonLight
.getSpecies();
310 String subSpeciesStr
= taxonLight
.getSubspecies();
311 String homotypicSynonymsString
= taxonLight
.getHomotypicSynonyms();
312 List
<String
> homotypicSynonymList
= Arrays
.asList(homotypicSynonymsString
.split(";"));
313 String heterotypicSynonymsString
= taxonLight
.getHeterotypicSynonyms();
314 List
<String
> heterotypicSynonymList
= Arrays
.asList(heterotypicSynonymsString
.split(";"));
316 String systematicsString
= taxonLight
.getSystematics();
317 String endemismString
= taxonLight
.getEndemism();
318 String statusString
= taxonLight
.getStatus();
319 String redBookCategory
= taxonLight
.getRedDataBookCategory();
321 if (StringUtils
.isNotBlank(speciesStr
)) {
322 boolean speciesIsExisting
= false;
323 Taxon mainTaxon
= null;
325 Taxon speciesTaxon
= (Taxon
)createTaxon(state
, Rank
.SPECIES(), speciesStr
, Taxon
.class, nc
);
326 mainTaxon
= speciesTaxon
;
329 if (StringUtils
.isNotBlank(subSpeciesStr
)){
330 Taxon existingSpecies
= state
.getHigherTaxon(speciesStr
);
331 if (existingSpecies
!= null){
332 speciesIsExisting
= true;
333 speciesTaxon
= existingSpecies
;
336 Taxon subSpeciesTaxon
= (Taxon
)createTaxon(state
, Rank
.SUBSPECIES(), subSpeciesStr
, Taxon
.class, nc
);
338 if (subSpeciesTaxon
!= null){
339 makeParent(state
, speciesTaxon
, subSpeciesTaxon
, citation
, microCitation
);
341 mainTaxon
= subSpeciesTaxon
;
342 state
.putHigherTaxon(speciesStr
, speciesTaxon
);
345 if (! speciesIsExisting
){
346 makeHigherTaxa(state
, taxonLight
, speciesTaxon
, citation
, microCitation
);
348 makeHomotypicSynonyms(state
, citation
, microCitation
, homotypicSynonymList
, mainTaxon
);
349 makeHeterotypicSynonyms(state
, citation
, microCitation
, heterotypicSynonymList
, mainTaxon
);
350 makeSystematics(systematicsString
, mainTaxon
);
351 makeEndemism(endemismString
, mainTaxon
);
352 makeStatus(statusString
, mainTaxon
);
353 makeRedBookCategory(redBookCategory
, mainTaxon
);
355 // state.putHigherTaxon(higherName, uuid);//(speciesStr, mainTaxon);
356 getTaxonService().save(mainTaxon
);
362 private void makeHigherTaxa(CyprusImportState state
, CyprusRow taxonLight
, Taxon speciesTaxon
, Reference citation
, String microCitation
) {
363 String divisionStr
= taxonLight
.getDivision();
364 String genusStr
= taxonLight
.getGenus();
365 String familyStr
= taxonLight
.getFamily();
367 Taxon division
= getTaxon(state
, divisionStr
, Rank
.DIVISION(), null, citation
, microCitation
);
368 Taxon family
= getTaxon(state
, familyStr
, Rank
.FAMILY(), division
, citation
, microCitation
);
369 Taxon genus
= getTaxon(state
, genusStr
, Rank
.GENUS(), family
, citation
, microCitation
);
370 makeParent(state
, genus
, speciesTaxon
, citation
, microCitation
) ;
374 private Taxon
getTaxon(CyprusImportState state
, String taxonNameStr
, Rank rank
, Taxon parent
, Reference citation
, String microCitation
) {
376 if (state
.containsHigherTaxon(taxonNameStr
)){
377 result
= state
.getHigherTaxon(taxonNameStr
);
379 result
= (Taxon
)createTaxon(state
, rank
, taxonNameStr
, Taxon
.class, nc
);
380 state
.putHigherTaxon(taxonNameStr
, result
);
382 makeParent(state
, null,result
, citation
, microCitation
);
384 makeParent(state
, parent
, result
, citation
, microCitation
);
392 private void makeHomotypicSynonyms(CyprusImportState state
,
393 Reference citation
, String microCitation
, List
<String
> homotypicSynonymList
, Taxon mainTaxon
) {
394 for (String homotypicSynonym
: homotypicSynonymList
){
395 if (StringUtils
.isNotBlank(homotypicSynonym
)){
396 Synonym synonym
= (Synonym
)createTaxon(state
, null, homotypicSynonym
, Synonym
.class, nc
);
397 mainTaxon
.addHomotypicSynonym(synonym
, citation
, microCitation
);
403 private void makeHeterotypicSynonyms(CyprusImportState state
, Reference citation
, String microCitation
, List
<String
> heterotypicSynonymList
, Taxon mainTaxon
) {
404 for (String heterotypicSynonym
: heterotypicSynonymList
){
405 if (StringUtils
.isNotBlank(heterotypicSynonym
)){
406 Synonym synonym
= (Synonym
)createTaxon(state
, null, heterotypicSynonym
, Synonym
.class, nc
);
407 mainTaxon
.addSynonym(synonym
, SynonymRelationshipType
.HETEROTYPIC_SYNONYM_OF(), citation
, microCitation
);
413 private void makeSystematics(String systematicsString
, Taxon mainTaxon
) {
415 if (StringUtils
.isNotBlank(systematicsString
)){
416 TaxonDescription td
= this.getTaxonDescription(mainTaxon
, false, true);
417 TextData textData
= TextData
.NewInstance(Feature
.SYSTEMATICS());
418 textData
.putText(systematicsString
, Language
.UNDETERMINED());
419 td
.addElement(textData
);
424 private void makeEndemism(String endemismString
, Taxon mainTaxon
) {
426 if (StringUtils
.isNotBlank(endemismString
)){
427 //OLD - not wanted as marker
429 // if (endemismString.trim().equalsIgnoreCase("not endemic") || endemismString.trim().equalsIgnoreCase("ne?")){
431 // }else if (endemismString.trim().equalsIgnoreCase("endemic")){
434 // throw new RuntimeException(endemismString + " is not a valid value for endemism");
436 // Marker marker = Marker.NewInstance(MarkerType.ENDEMIC(), flag);
437 // mainTaxon.addMarker(marker);
439 TaxonDescription td
= this.getTaxonDescription(mainTaxon
, false, true);
440 TextData textData
= TextData
.NewInstance(endemism
);
441 textData
.putText(endemismString
, Language
.ENGLISH());
442 td
.addElement(textData
);
447 private void makeStatus(String statusString
, Taxon mainTaxon
) {
449 if (StringUtils
.isNotBlank(statusString
)){
450 PresenceTerm status
= null;
451 if (statusString
.contains("Indigenous?")){
452 status
= indigenousDoubtful
;
453 }else if (statusString
.contains("Indigenous")){
455 }else if (statusString
.contains("Casual?") || statusString
.contains("Causal?")){
456 status
= casualDoubtful
;
457 }else if (statusString
.contains("Casual")){
459 }else if (statusString
.contains("Cultivated?")){
460 status
= cultivatedDoubtful
;
461 }else if (statusString
.contains("Cultivated")){
462 status
= PresenceTerm
.CULTIVATED();
463 }else if (statusString
.contains("non-invasive?")){
464 status
= nonInvasiveDoubtful
;
465 }else if (statusString
.contains("non-invasive")){
466 status
= nonInvasive
;
467 }else if (statusString
.contains("invasive?")){
468 status
= invasiveDoubtful
;
469 }else if (statusString
.contains("invasive")){
471 }else if (statusString
.contains("Questionable?")){
472 status
= questionableDoubtful
;
473 }else if (statusString
.contains("Questionable")){
474 status
= questionable
;
475 }else if (statusString
.startsWith("F")){
477 }else if (statusString
.equals("##")){
480 logger
.warn("Unknown status: " + statusString
);
483 TaxonDescription td
= this.getTaxonDescription(mainTaxon
, false, true);
484 NamedArea area
= TdwgArea
.getAreaByTdwgAbbreviation("CYP");
485 Distribution distribution
= Distribution
.NewInstance(area
, status
);
486 td
.addElement(distribution
);
489 TextData textData
= TextData
.NewInstance(Feature
.STATUS());
490 textData
.putText(statusString
, Language
.ENGLISH());
491 td
.addElement(textData
);
496 private void makeRedBookCategory(String redBookCategory
, Taxon mainTaxon
) {
497 //red data book category
498 if (StringUtils
.isNotBlank(redBookCategory
)){
499 TaxonDescription td
= this.getTaxonDescription(mainTaxon
, false, true);
500 TextData textData
= TextData
.NewInstance(this.redBookCategory
);
501 textData
.putText(redBookCategory
, Language
.ENGLISH());
502 td
.addElement(textData
);
510 * Stores parent-child, synonym and common name relationships
513 protected boolean secondPass(CyprusImportState state
) {
514 boolean success
= true;
515 // CyprusRow cyprusRow = state.getCyprusRow();
525 * @param taxonNameStr
531 private TaxonBase
createTaxon(CyprusImportState state
, Rank rank
, String taxonNameStr
,
532 Class statusClass
, NomenclaturalCode nc
) {
534 NonViralName taxonNameBase
= null;
535 if (nc
== NomenclaturalCode
.ICVCN
){
536 logger
.warn("ICVCN not yet supported");
539 taxonNameBase
=(NonViralName
) nc
.getNewTaxonNameInstance(rank
);
540 //NonViralName nonViralName = (NonViralName)taxonNameBase;
541 INonViralNameParser parser
= nameParser
;//NonViralNameParserImpl.NewInstance();
542 taxonNameBase
= (NonViralName
<BotanicalName
>)parser
.parseFullName(taxonNameStr
, nc
, rank
);
544 //taxonNameBase.setNameCache(taxonNameStr);
549 Reference sec
= state
.getConfig().getSourceReference();
551 if (statusClass
.equals(Taxon
.class)){
552 taxonBase
= Taxon
.NewInstance(taxonNameBase
, sec
);
553 }else if (statusClass
.equals(Synonym
.class)){
554 taxonBase
= Synonym
.NewInstance(taxonNameBase
, sec
);
556 Taxon taxon
= Taxon
.NewInstance(taxonNameBase
, sec
);
557 taxon
.setTaxonStatusUnknown(true);
563 private boolean makeParent(CyprusImportState state
, Taxon parentTaxon
, Taxon childTaxon
, Reference citation
, String microCitation
){
564 boolean success
= true;
565 Reference sec
= state
.getConfig().getSourceReference();
567 // Reference sec = parentTaxon.getSec();
568 Classification tree
= state
.getTree(sec
);
570 tree
= makeTree(state
, sec
);
571 tree
.setTitleCache(state
.getConfig().getSourceReferenceTitle());
573 if (sec
.equals(childTaxon
.getSec())){
574 success
&= (null != tree
.addParentChild(parentTaxon
, childTaxon
, citation
, microCitation
));
576 logger
.warn("No relationship added for child " + childTaxon
.getTitleCache());