2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.faunaEuropaea
;
12 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.A_AUCT
;
13 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.P_PARENTHESIS
;
14 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.Q_NO_RESTRICTION
;
15 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.R_GENUS
;
16 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.R_SUBGENUS
;
17 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.R_SPECIES
;
18 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.R_SUBSPECIES
;
19 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.T_STATUS_ACCEPTED
;
20 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.T_STATUS_NOT_ACCEPTED
;
22 import java
.sql
.ResultSet
;
23 import java
.sql
.SQLException
;
24 import java
.util
.Collection
;
25 import java
.util
.HashMap
;
27 import java
.util
.UUID
;
29 import org
.apache
.log4j
.Logger
;
30 import org
.springframework
.stereotype
.Component
;
31 import org
.springframework
.transaction
.TransactionStatus
;
33 import eu
.etaxonomy
.cdm
.common
.CdmUtils
;
34 import eu
.etaxonomy
.cdm
.io
.berlinModel
.CdmOneToManyMapper
;
35 import eu
.etaxonomy
.cdm
.io
.berlinModel
.CdmStringMapper
;
36 import eu
.etaxonomy
.cdm
.io
.common
.CdmAttributeMapperBase
;
37 import eu
.etaxonomy
.cdm
.io
.common
.CdmSingleAttributeMapperBase
;
38 import eu
.etaxonomy
.cdm
.io
.common
.ICdmIO
;
39 import eu
.etaxonomy
.cdm
.io
.common
.IImportConfigurator
;
40 import eu
.etaxonomy
.cdm
.io
.common
.ImportHelper
;
41 import eu
.etaxonomy
.cdm
.io
.common
.MapWrapper
;
42 import eu
.etaxonomy
.cdm
.io
.common
.Source
;
43 import eu
.etaxonomy
.cdm
.model
.agent
.TeamOrPersonBase
;
44 import eu
.etaxonomy
.cdm
.model
.common
.CdmBase
;
45 import eu
.etaxonomy
.cdm
.model
.name
.Rank
;
46 import eu
.etaxonomy
.cdm
.model
.name
.TaxonNameBase
;
47 import eu
.etaxonomy
.cdm
.model
.name
.ZoologicalName
;
48 import eu
.etaxonomy
.cdm
.model
.reference
.PublicationBase
;
49 import eu
.etaxonomy
.cdm
.model
.reference
.Publisher
;
50 import eu
.etaxonomy
.cdm
.model
.reference
.ReferenceBase
;
51 import eu
.etaxonomy
.cdm
.model
.taxon
.Synonym
;
52 import eu
.etaxonomy
.cdm
.model
.taxon
.Taxon
;
53 import eu
.etaxonomy
.cdm
.model
.taxon
.TaxonBase
;
54 import eu
.etaxonomy
.cdm
.strategy
.exceptions
.UnknownCdmTypeException
;
58 * @author a.babadshanjan
63 public class FaunaEuropaeaTaxonImport
extends FaunaEuropaeaImportBase
{
64 private static final Logger logger
= Logger
.getLogger(FaunaEuropaeaTaxonImport
.class);
66 /* Max number of taxa to retrieve (for test purposes) */
67 private int maxTaxa
= 5000;
68 /* Max number of taxa to be saved with one service call */
69 private int limit
= 10000; // TODO: Make configurable
70 /* Interval for progress info message when retrieving taxa */
71 private int modCount
= 10000;
72 /* Highest taxon index in the FauEu database */
73 private int highestTaxonIndex
= 0;
74 /* Number of times method buildParentName() has been called for one taxon */
75 private int callCount
= 0;
76 private Map
<Integer
, FaunaEuropaeaTaxon
> fauEuTaxonMap
= new HashMap();
80 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)
83 protected boolean doCheck(IImportConfigurator config
) {
84 boolean result
= true;
85 FaunaEuropaeaImportConfigurator fauEuConfig
= (FaunaEuropaeaImportConfigurator
)config
;
86 logger
.warn("Checking for Taxa not yet fully implemented");
87 result
&= checkTaxonStatus(fauEuConfig
);
93 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
95 protected boolean isIgnore(IImportConfigurator config
) {
96 return !config
.isDoTaxa();
99 private boolean checkTaxonStatus(FaunaEuropaeaImportConfigurator fauEuConfig
) {
100 boolean result
= true;
102 Source source
= fauEuConfig
.getSource();
104 ResultSet rs
= source
.getResultSet(sqlStr
);
106 // } catch (SQLException e) {
107 // e.printStackTrace();
113 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IImportConfigurator, eu.etaxonomy.cdm.api.application.CdmApplicationController, java.util.Map)
116 protected boolean doInvoke(IImportConfigurator config
,
117 Map
<String
, MapWrapper
<?
extends CdmBase
>> stores
) {
119 MapWrapper
<TaxonNameBase
<?
,?
>> taxonNamesStore
= (MapWrapper
<TaxonNameBase
<?
,?
>>)stores
.get(ICdmIO
.TAXONNAME_STORE
);
120 // Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap = new HashMap();
121 FaunaEuropaeaImportConfigurator fauEuConfig
= (FaunaEuropaeaImportConfigurator
)config
;
122 boolean success
= true;
124 if(logger
.isInfoEnabled()) { logger
.info("Start making taxa..."); }
126 success
= retrieveTaxa(fauEuConfig
, stores
, fauEuTaxonMap
, Q_NO_RESTRICTION
);
127 success
= processTaxaSecondPass(fauEuConfig
, stores
, fauEuTaxonMap
);
128 success
= saveTaxa(stores
);
130 logger
.info("End making taxa...");
135 private boolean retrieveTaxa(FaunaEuropaeaImportConfigurator fauEuConfig
,
136 Map
<String
, MapWrapper
<?
extends CdmBase
>> stores
,
137 Map
<Integer
, FaunaEuropaeaTaxon
> fauEuTaxonMap
, int valid
) {
139 MapWrapper
<TaxonBase
> taxonStore
= (MapWrapper
<TaxonBase
>)stores
.get(ICdmIO
.TAXON_STORE
);
141 Source source
= fauEuConfig
.getSource();
142 String namespace
= "Taxon";
144 boolean success
= true;
149 " SELECT MAX(TAX_ID) AS TAX_ID FROM dbo.Taxon ";
151 ResultSet rs
= source
.getResultSet(strQuery
);
153 int maxTaxonId
= rs
.getInt("TAX_ID");
154 highestTaxonIndex
= maxTaxonId
;
159 top
= "TOP " + maxTaxa
;
162 String validClause
= "";
163 if (valid
== T_STATUS_ACCEPTED
|| valid
== T_STATUS_NOT_ACCEPTED
) {
164 validClause
= " AND " + " TAX_VALID = " + valid
;
168 " SELECT " + top
+ " Taxon.*, rank.*, author.* " +
170 " LEFT OUTER JOIN dbo.author ON dbo.Taxon.TAX_AUT_ID = dbo.author.aut_id " +
171 " LEFT OUTER JOIN dbo.rank ON dbo.Taxon.TAX_RNK_ID = dbo.rank.rnk_id " +
175 if (logger
.isInfoEnabled()) {
176 logger
.info("Query: " + strQuery
);
178 rs
= source
.getResultSet(strQuery
);
182 if ((i
++ % modCount
) == 0 && i
!= 1 ) {
183 if(logger
.isInfoEnabled()) {
184 logger
.info("Taxa retrieved: " + (i
-1));
188 int taxonId
= rs
.getInt("TAX_ID");
189 String localName
= rs
.getString("TAX_NAME");
190 int rankId
= rs
.getInt("TAX_RNK_ID");
191 int parentId
= rs
.getInt("TAX_TAX_IDPARENT");
192 int familyId
= rs
.getInt("TAX_TAX_IDFAMILY");
193 int genusId
= rs
.getInt("TAX_TAX_IDGENUS");
194 int autId
= rs
.getInt("TAX_AUT_ID");
195 int status
= rs
.getInt("TAX_VALID");
196 int year
= rs
.getInt("TAX_YEAR");
197 int parenthesis
= rs
.getInt("TAX_PARENTHESIS");
198 String autName
= rs
.getString("aut_name");
200 UUID taxonBaseUuid
= UUID
.randomUUID();
203 rank
= FaunaEuropaeaTransformer
.rankId2Rank(rs
, false);
204 } catch (UnknownCdmTypeException e
) {
205 logger
.warn("Taxon (" + taxonId
+ ") has unknown rank (" + rankId
+ ") and could not be saved.");
207 } catch (NullPointerException e
) {
208 logger
.warn("Taxon (" + taxonId
+ ") has rank null and could not be saved.");
212 ReferenceBase
<?
> reference
= null;
214 ZoologicalName zooName
= ZoologicalName
.NewInstance(rank
);
215 String nameTitleCache
= localName
;
217 // set local name cache
218 zooName
.setNameCache(localName
);
220 TaxonBase
<?
> taxonBase
;
221 FaunaEuropaeaTaxon fauEuTaxon
= new FaunaEuropaeaTaxon();
226 if ((status
== T_STATUS_ACCEPTED
) || (autId
== A_AUCT
)) {
227 taxon
= Taxon
.NewInstance(zooName
, reference
);
229 } else if ((status
== T_STATUS_NOT_ACCEPTED
) && (autId
!= A_AUCT
)) {
230 synonym
= Synonym
.NewInstance(zooName
, reference
);
233 logger
.warn("Unknown taxon status " + status
+ ". Taxon (" + taxonId
+ ") ignored.");
237 taxonBase
.setUuid(taxonBaseUuid
);
239 fauEuTaxon
.setUuid(taxonBaseUuid
);
240 fauEuTaxon
.setLocalName(localName
);
241 fauEuTaxon
.setParentId(parentId
);
242 fauEuTaxon
.setId(taxonId
);
243 fauEuTaxon
.setRankId(rankId
);
244 fauEuTaxon
.setYear(year
);
245 fauEuTaxon
.setAuthor(autName
);
246 if (parenthesis
== P_PARENTHESIS
) {
247 fauEuTaxon
.setParenthesis(true);
249 fauEuTaxon
.setParenthesis(false);
251 if (status
== T_STATUS_ACCEPTED
) {
252 fauEuTaxon
.setValid(true);
254 fauEuTaxon
.setValid(false);
257 ImportHelper
.setOriginalSource(taxonBase
, fauEuConfig
.getSourceReference(), taxonId
, namespace
);
259 if (!taxonStore
.containsId(taxonId
)) {
260 if (taxonBase
== null) {
261 logger
.warn("Taxon base is null");
263 taxonStore
.put(taxonId
, taxonBase
);
264 fauEuTaxonMap
.put(taxonId
, fauEuTaxon
);
265 // if (logger.isDebugEnabled()) {
266 // logger.debug("Stored taxon base (" + taxonId + ") " + localName);
269 logger
.warn("Not imported taxon base with duplicated TAX_ID (" + taxonId
+
272 } catch (Exception e
) {
273 logger
.warn("An exception occurred when creating taxon base with id " + taxonId
+
274 ". Taxon base could not be saved.");
277 } catch (SQLException e
) {
278 logger
.error("SQLException:" + e
);
286 private boolean processTaxaSecondPass(FaunaEuropaeaImportConfigurator fauEuConfig
,
287 Map
<String
, MapWrapper
<?
extends CdmBase
>> stores
,
288 Map
<Integer
, FaunaEuropaeaTaxon
> fauEuTaxonMap
) {
290 if(logger
.isInfoEnabled()) { logger
.info("Processing taxa second pass..."); }
292 MapWrapper
<TaxonBase
> taxonStore
= (MapWrapper
<TaxonBase
>)stores
.get(ICdmIO
.TAXON_STORE
);
294 boolean success
= true;
296 for (int id
: taxonStore
.keySet())
298 TaxonBase
<?
> taxonBase
= taxonStore
.get(id
);
299 FaunaEuropaeaTaxon fauEuTaxon
= fauEuTaxonMap
.get(id
);
301 if (logger
.isInfoEnabled()) { logger
.info("Taxon # " + id
); }
302 String nameString
= calculateTaxonName(fauEuTaxon
, taxonBase
, taxonStore
, fauEuTaxonMap
);
303 // String nameString = calculateTaxonName(fauEuTaxon, taxonBase, taxonStore);
304 setTaxonName(nameString
, fauEuTaxon
, taxonBase
);
311 /* Remove last part of name */
312 private String
removeEpithet(String nameString
) {
314 String subString
= nameString
;
315 int index
= nameString
.lastIndexOf(" ");
317 subString
= nameString
.substring(0, index
);
321 // String[] tokens = nameString.split("[\\s]+");
322 // StringBuilder stringBuilder =
323 // int len = tokens.length - 1;
324 // for (int i = 0; i < len - 1; i++) {
325 // String lastToken = tokens[len];
331 /* Remove last part of name */
332 private String
exchangeEpithet(String nameString
) {
334 String subString
= nameString
;
335 int index
= nameString
.lastIndexOf(" ");
337 subString
= nameString
.substring(0, index
);
341 // String[] tokens = nameString.split("[\\s]+");
342 // StringBuilder stringBuilder =
343 // int len = tokens.length - 1;
344 // for (int i = 0; i < len - 1; i++) {
345 // String lastToken = tokens[len];
351 /* Build name title cache */
352 private String
buildNameTitleCache(String nameString
, FaunaEuropaeaTaxon fauEuTaxon
) {
354 StringBuilder titleCacheStringBuilder
= new StringBuilder(nameString
);
355 int year
= fauEuTaxon
.getYear();
356 if (year
!= 0) { // TODO: What do do with authors like xp, xf, etc?
357 titleCacheStringBuilder
.append(" ");
358 if (fauEuTaxon
.isParenthesis() == true) {
359 titleCacheStringBuilder
.append("(");
361 titleCacheStringBuilder
.append(fauEuTaxon
.getAuthor());
362 titleCacheStringBuilder
.append(" ");
363 titleCacheStringBuilder
.append(year
);
364 if (fauEuTaxon
.isParenthesis() == true) {
365 titleCacheStringBuilder
.append(")");
368 return titleCacheStringBuilder
.toString();
372 /* For a given rank, returns max number of recurse calls of buildParentName() */
373 public static int maxCallsPerRank(int rankId
) {
396 /* Build parent's taxon name */
397 private String
buildParentName(int originalTaxonRank
, String concatString
, FaunaEuropaeaTaxon fauEuTaxon
,
398 MapWrapper
<TaxonBase
> taxonStore
) {
402 /* Concatenated name string of parent */
403 // String parentConcatString = "";
404 String parentConcatString
= concatString
;
406 StringBuilder parentConcatStringBuilder
= new StringBuilder();
407 /* Local name of parent */
408 String parentString
= null;
410 int parentId
= fauEuTaxon
.getParentId();
411 int rankId
= fauEuTaxon
.getRankId();
412 FaunaEuropaeaTaxon parent
= fauEuTaxonMap
.get(parentId
);
413 // TaxonBase<?> parentTaxonBase = null;
414 // TaxonNameBase<?,?> parentName = null;
416 if (parent
== null) {
417 logger
.warn("Parent of taxon (" + fauEuTaxon
.getId() + ") is null");
418 return parentConcatString
;
420 if (logger
.isDebugEnabled()) {
421 logger
.debug("Concat string: " + concatString
);
424 if ((rankId
== R_SPECIES
|| rankId
== R_SUBSPECIES
) || (rankId
== R_SUBGENUS
&& callCount
> 1)) {
425 UUID parentUuid
= parent
.getUuid();
426 if (parentUuid
!= null) {
427 // parentTaxonBase = taxonStore.get(parentId);
428 // if (parentTaxonBase != null) {
429 // parentName = parentTaxonBase.getName();
430 // if(parentName != null) {
431 // parentString = ((ZoologicalName)parentName).getNameCache();
434 parentString
= parent
.getLocalName();
436 // if (fauEuTaxon.isValid()) {
437 // parentString = parent.getLocalName();
439 // parentString = fauEuTaxon.get
442 // parentString is either already complete or just local, depending on whether the
443 // parent taxon has already processed
444 if (logger
.isDebugEnabled()) {
445 logger
.debug("Parent string: " + parentString
);
447 if (!fauEuTaxon
.isValid()) {
448 // parentString = exchangeEpithet(parentString);
450 if (logger
.isDebugEnabled()) {
451 logger
.debug("Emptied synonym's parent string");
455 // logger.warn("Parent taxon name of taxon (uuid= " + parentUuid.toString() + "), id = " +
456 // parent.getId() + ") is null");
459 // logger.warn("Parent taxon (uuid= " + parentUuid.toString() + "), id = " +
460 // parent.getId() + ") is null");
462 // if (parent.isNameComplete()) {
463 // // make sure to stop parent name building
466 // if (parent.getRankId() == R_SUBGENUS && !parent.isNameComplete()) {
467 if (parent
.getRankId() == R_SUBGENUS
) {
468 parentConcatStringBuilder
.append("(");
470 parentConcatStringBuilder
.append(parentString
);
471 // if (parent.getRankId() == R_SUBGENUS && !parent.isNameComplete()) {
472 if (parent
.getRankId() == R_SUBGENUS
) {
473 parentConcatStringBuilder
.append(")");
475 parentConcatStringBuilder
.append(" ");
476 parentConcatStringBuilder
.append(concatString
);
477 parentConcatString
= parentConcatStringBuilder
.toString();
479 if (logger
.isInfoEnabled()) {
480 logger
.info("Concatenated name: " + parentConcatString
);
483 logger
.warn("Parent uuid of " + parentId
+ " is null");
486 } else { // Higher ranks
487 if (logger
.isDebugEnabled()) {
488 logger
.debug("Name complete for taxon rank (" + rankId
+ ")");
490 return parentConcatString
;
492 int maxCalls
= maxCallsPerRank(originalTaxonRank
);
493 if (logger
.isDebugEnabled()) {
494 logger
.debug("Call counter: " + callCount
);
495 logger
.debug("Max calls: " + maxCalls
);
497 if (callCount
<= maxCalls
) {
498 if ((rankId
== R_SPECIES
|| rankId
== R_SUBSPECIES
) || (rankId
== R_SUBGENUS
&& callCount
> 1)) {
499 parentConcatString
= buildParentName(originalTaxonRank
, parentConcatString
, parent
, taxonStore
);
503 return parentConcatString
;
507 /* Build taxon's concatenated name string */
508 private String
calculateTaxonName(FaunaEuropaeaTaxon fauEuTaxon
,
509 TaxonBase
<?
> taxonBase
, MapWrapper
<TaxonBase
> taxonStore
,
510 Map
<Integer
, FaunaEuropaeaTaxon
> fauEuTaxonMap
) {
512 /* Local name string */
513 String localString
= "";
514 /* Concatenated parent's name string */
515 String parentString
= "";
517 TaxonNameBase
<?
,?
> taxonName
= taxonBase
.getName();
518 ZoologicalName zooName
= (ZoologicalName
)taxonName
;
520 if (zooName
!= null) {
521 localString
= zooName
.getNameCache();
524 int rank
= fauEuTaxon
.getRankId();
526 if(logger
.isInfoEnabled()) {
527 logger
.info("Local taxon name (rank = " + rank
+ "): " + localString
);
531 parentString
= buildParentName(rank
, localString
, fauEuTaxon
, taxonStore
);
532 parentString
= (String
) CdmUtils
.removeDuplicateWhitespace(parentString
.trim());
537 /* Build taxon name */
538 private boolean setTaxonName(String concatString
, FaunaEuropaeaTaxon fauEuTaxon
,
539 TaxonBase
<?
> taxonBase
) {
541 boolean success
= true;
543 TaxonNameBase
<?
,?
> taxonName
= taxonBase
.getName();
544 ZoologicalName zooName
= (ZoologicalName
)taxonName
;
546 zooName
.setNameCache(concatString
);
547 String titleCache
= buildNameTitleCache(concatString
, fauEuTaxon
);
548 zooName
.setTitleCache(titleCache
);
549 zooName
.setFullTitleCache(titleCache
); // TODO: Add reference, NC status
551 // Set the complete scientific name in FaunaEuropaeaTaxon,
552 // including parent(s) parts.
553 // fauEuTaxon.setScientificName(concatString);
554 fauEuTaxon
.setNameComplete(true);
556 if (logger
.isInfoEnabled()) {
557 logger
.info("Name stored: " + concatString
);
563 private boolean saveTaxa(Map
<String
, MapWrapper
<?
extends CdmBase
>> stores
) {
565 MapWrapper
<TaxonBase
> taxonStore
= (MapWrapper
<TaxonBase
>)stores
.get(ICdmIO
.TAXON_STORE
);
568 int nbrOfTaxa
= highestTaxonIndex
;
569 // int nbrOfTaxa = taxonStore.size();
570 boolean success
= true;
572 if(logger
.isInfoEnabled()) { logger
.info("Saving taxa ..."); }
574 if (nbrOfTaxa
< limit
) { // TODO: test with critical values
577 n
= nbrOfTaxa
/ limit
;
580 if(logger
.isInfoEnabled()) {
581 logger
.info("number of taxa = " + taxonStore
.size()
582 + ", highest taxon index = " + highestTaxonIndex
583 + ", limit = " + limit
587 // save taxa in chunks of <=limit
589 for (int j
= 1; j
<= n
+ 1; j
++)
592 int start
= offset
* limit
;
594 if(logger
.isInfoEnabled()) { logger
.info("Saving taxa: " + start
+ " - " + (start
+ limit
- 1)); }
596 if(logger
.isInfoEnabled()) {
597 logger
.info("index = " + j
598 + ", offset = " + offset
599 + ", start = " + start
);
603 limit
= nbrOfTaxa
- n
* limit
;
604 if(logger
.isInfoEnabled()) { logger
.info("n = " + n
+ " limit = " + limit
); }
607 Collection
<TaxonBase
> taxonMapPart
= taxonStore
.objects(start
, limit
);
608 getTaxonService().saveTaxonAll(taxonMapPart
);
610 taxonStore
.removeObjects(start
, limit
);