2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.faunaEuropaea
;
12 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.A_AUCT
;
13 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.P_PARENTHESIS
;
14 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.R_GENUS
;
15 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.R_SUBGENUS
;
16 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.T_STATUS_ACCEPTED
;
17 import static eu
.etaxonomy
.cdm
.io
.faunaEuropaea
.FaunaEuropaeaTransformer
.T_STATUS_NOT_ACCEPTED
;
19 import java
.sql
.ResultSet
;
20 import java
.sql
.SQLException
;
21 import java
.util
.Collection
;
22 import java
.util
.HashMap
;
24 import java
.util
.UUID
;
26 import org
.apache
.log4j
.Logger
;
27 import org
.springframework
.stereotype
.Component
;
28 import org
.springframework
.transaction
.TransactionStatus
;
30 import eu
.etaxonomy
.cdm
.common
.CdmUtils
;
31 import eu
.etaxonomy
.cdm
.io
.berlinModel
.CdmOneToManyMapper
;
32 import eu
.etaxonomy
.cdm
.io
.berlinModel
.CdmStringMapper
;
33 import eu
.etaxonomy
.cdm
.io
.common
.CdmAttributeMapperBase
;
34 import eu
.etaxonomy
.cdm
.io
.common
.CdmSingleAttributeMapperBase
;
35 import eu
.etaxonomy
.cdm
.io
.common
.ICdmIO
;
36 import eu
.etaxonomy
.cdm
.io
.common
.IImportConfigurator
;
37 import eu
.etaxonomy
.cdm
.io
.common
.ImportHelper
;
38 import eu
.etaxonomy
.cdm
.io
.common
.MapWrapper
;
39 import eu
.etaxonomy
.cdm
.io
.common
.Source
;
40 import eu
.etaxonomy
.cdm
.model
.agent
.TeamOrPersonBase
;
41 import eu
.etaxonomy
.cdm
.model
.common
.CdmBase
;
42 import eu
.etaxonomy
.cdm
.model
.name
.Rank
;
43 import eu
.etaxonomy
.cdm
.model
.name
.TaxonNameBase
;
44 import eu
.etaxonomy
.cdm
.model
.name
.ZoologicalName
;
45 import eu
.etaxonomy
.cdm
.model
.reference
.PublicationBase
;
46 import eu
.etaxonomy
.cdm
.model
.reference
.Publisher
;
47 import eu
.etaxonomy
.cdm
.model
.reference
.ReferenceBase
;
48 import eu
.etaxonomy
.cdm
.model
.taxon
.Synonym
;
49 import eu
.etaxonomy
.cdm
.model
.taxon
.Taxon
;
50 import eu
.etaxonomy
.cdm
.model
.taxon
.TaxonBase
;
51 import eu
.etaxonomy
.cdm
.strategy
.exceptions
.UnknownCdmTypeException
;
55 * @author a.babadshanjan
60 public class FaunaEuropaeaTaxonImport
extends FaunaEuropaeaImportBase
{
61 private static final Logger logger
= Logger
.getLogger(FaunaEuropaeaTaxonImport
.class);
63 /* Interval for progress info message when retrieving taxa */
64 private int modCount
= 10000;
65 /* Max number of taxa to be saved with one service call */
66 private int limit
= 10000; // TODO: Make configurable
67 /* The highest taxon index in the FauEu database */
68 private int highestTaxonIndex
= 0;
69 private Map
<Integer
, FaunaEuropaeaTaxon
> fauEuTaxonMap
= new HashMap();
73 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IImportConfigurator)
76 protected boolean doCheck(IImportConfigurator config
) {
77 boolean result
= true;
78 FaunaEuropaeaImportConfigurator fauEuConfig
= (FaunaEuropaeaImportConfigurator
)config
;
79 logger
.warn("Checking for Taxa not yet fully implemented");
80 result
&= checkTaxonStatus(fauEuConfig
);
86 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
88 protected boolean isIgnore(IImportConfigurator config
) {
89 return !config
.isDoTaxa();
92 private boolean checkTaxonStatus(FaunaEuropaeaImportConfigurator fauEuConfig
) {
93 boolean result
= true;
95 Source source
= fauEuConfig
.getSource();
97 ResultSet rs
= source
.getResultSet(sqlStr
);
99 // } catch (SQLException e) {
100 // e.printStackTrace();
106 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IImportConfigurator, eu.etaxonomy.cdm.api.application.CdmApplicationController, java.util.Map)
109 protected boolean doInvoke(IImportConfigurator config
,
110 Map
<String
, MapWrapper
<?
extends CdmBase
>> stores
) {
112 MapWrapper
<TaxonNameBase
<?
,?
>> taxonNamesStore
= (MapWrapper
<TaxonNameBase
<?
,?
>>)stores
.get(ICdmIO
.TAXONNAME_STORE
);
113 // Map<Integer, FaunaEuropaeaTaxon> fauEuTaxonMap = new HashMap();
114 FaunaEuropaeaImportConfigurator fauEuConfig
= (FaunaEuropaeaImportConfigurator
)config
;
115 boolean success
= true;
117 if(logger
.isInfoEnabled()) { logger
.info("Start making taxa..."); }
119 success
= retrieveTaxa(fauEuConfig
, stores
, fauEuTaxonMap
);
120 success
= processTaxaSecondPass(fauEuConfig
, stores
, fauEuTaxonMap
);
121 success
= saveTaxa(stores
);
123 logger
.info("End making taxa...");
128 private boolean retrieveTaxa(FaunaEuropaeaImportConfigurator fauEuConfig
,
129 Map
<String
, MapWrapper
<?
extends CdmBase
>> stores
,
130 Map
<Integer
, FaunaEuropaeaTaxon
> fauEuTaxonMap
) {
132 // MapWrapper<TeamOrPersonBase> authorStore = (MapWrapper<TeamOrPersonBase>)stores.get(ICdmIO.TEAM_STORE);
133 // MapWrapper<ReferenceBase> refStore = (MapWrapper<ReferenceBase>)stores.get(ICdmIO.NOMREF_STORE);
134 // MapWrapper<TaxonNameBase<?,?>> taxonNamesStore = (MapWrapper<TaxonNameBase<?,?>>)stores.get(ICdmIO.TAXONNAME_STORE);
135 MapWrapper
<TaxonBase
> taxonStore
= (MapWrapper
<TaxonBase
>)stores
.get(ICdmIO
.TAXON_STORE
);
137 Source source
= fauEuConfig
.getSource();
138 String namespace
= "Taxon";
140 boolean success
= true;
145 " SELECT MAX(TAX_ID) AS TAX_ID FROM dbo.Taxon ";
147 ResultSet rs
= source
.getResultSet(strQuery
);
149 int maxTaxonId
= rs
.getInt("TAX_ID");
150 highestTaxonIndex
= maxTaxonId
;
154 // " SELECT Taxon.*, rank.* " +
155 // " FROM dbo.Taxon INNER JOIN dbo.rank ON dbo.Taxon.TAX_RNK_ID = dbo.rank.rnk_id " +
159 " SELECT Taxon.*, rank.*, author.* " +
160 " FROM dbo.Taxon INNER JOIN dbo.rank ON dbo.Taxon.TAX_RNK_ID = dbo.rank.rnk_id " +
161 " INNER JOIN dbo.author ON dbo.Taxon.TAX_AUT_ID = dbo.author.aut_id " +
164 rs
= source
.getResultSet(strQuery
);
170 if ((i
++ % modCount
) == 0 && i
!= 1 ) {
171 if(logger
.isInfoEnabled()) {
172 logger
.info("Taxa retrieved: " + (i
-1));
176 int taxonId
= rs
.getInt("TAX_ID");
177 String taxonName
= rs
.getString("TAX_NAME");
178 int rankId
= rs
.getInt("TAX_RNK_ID");
179 int parentId
= rs
.getInt("TAX_TAX_IDPARENT");
180 int familyId
= rs
.getInt("TAX_TAX_IDFAMILY");
181 int genusId
= rs
.getInt("TAX_TAX_IDGENUS");
182 int autId
= rs
.getInt("TAX_AUT_ID");
183 int status
= rs
.getInt("TAX_VALID");
184 int year
= rs
.getInt("TAX_YEAR");
185 int parenthesis
= rs
.getInt("TAX_PARENTHESIS");
186 String autName
= rs
.getString("aut_name");
188 UUID taxonBaseUuid
= UUID
.randomUUID();
191 rank
= FaunaEuropaeaTransformer
.rankId2Rank(rs
, false);
192 } catch (UnknownCdmTypeException e
) {
193 logger
.warn("Taxon (" + taxonId
+ ") has unknown rank (" + rankId
+ ") and could not be saved.");
197 ReferenceBase
<?
> reference
= null;
199 ZoologicalName zooName
= ZoologicalName
.NewInstance(rank
);
200 String nameTitleCache
= taxonName
;
202 // set local name cache
204 zooName
.setNameCache(taxonName
);
206 // StringBuilder nameTitleCacheBuilder = new StringBuilder(taxonName);
207 // if (year != 0) { // TODO: What do do with authors like xp, xf, etc?
208 // nameTitleCacheBuilder.append(" ");
209 // if (parenthesis == P_PARENTHESIS) {
210 // nameTitleCacheBuilder.append("(");
212 // nameTitleCacheBuilder.append(autName);
213 // nameTitleCacheBuilder.append(" ");
214 // nameTitleCacheBuilder.append(year);
215 // if (parenthesis == P_PARENTHESIS) {
216 // nameTitleCacheBuilder.append(")");
219 // nameTitleCache = nameTitleCacheBuilder.toString();
220 // zooName.setTitleCache(nameTitleCache);
221 // zooName.setFullTitleCache(nameTitleCache); // FIXME: reference, NC status
223 TaxonBase
<?
> taxonBase
;
224 FaunaEuropaeaTaxon fauEuTaxon
= new FaunaEuropaeaTaxon();
229 // logger.debug(status);
230 if ((status
== T_STATUS_ACCEPTED
) || (autId
== A_AUCT
)) {
231 taxon
= Taxon
.NewInstance(zooName
, reference
);
233 } else if ((status
== T_STATUS_NOT_ACCEPTED
) && (autId
!= A_AUCT
)) {
234 synonym
= Synonym
.NewInstance(zooName
, reference
);
237 logger
.warn("Unknown taxon status " + status
+ ". Taxon (" + taxonId
+ ") ignored.");
241 taxonBase
.setUuid(taxonBaseUuid
);
243 // set local title cache
245 // taxonBase.setTitleCache(nameTitleCache);
247 fauEuTaxon
.setUuid(taxonBaseUuid
);
248 fauEuTaxon
.setParentId(parentId
);
249 fauEuTaxon
.setId(taxonId
);
250 fauEuTaxon
.setRankId(rankId
);
251 fauEuTaxon
.setYear(year
);
252 fauEuTaxon
.setAuthor(autName
);
253 if (parenthesis
== P_PARENTHESIS
) {
254 fauEuTaxon
.setParenthesis(true);
256 fauEuTaxon
.setParenthesis(false);
259 ImportHelper
.setOriginalSource(taxonBase
, fauEuConfig
.getSourceReference(), taxonId
, namespace
);
261 if (!taxonStore
.containsId(taxonId
)) {
262 if (taxonBase
== null) {
263 logger
.warn("Taxon base is null");
265 taxonStore
.put(taxonId
, taxonBase
);
266 fauEuTaxonMap
.put(taxonId
, fauEuTaxon
);
267 if (logger
.isDebugEnabled()) {
268 logger
.debug("Stored taxon base (" + taxonId
+ ") " + taxonName
);
271 logger
.warn("Not imported taxon base with duplicated TAX_ID (" + taxonId
+
275 // if(!taxonNamesStore.containsId(taxonId) && !taxonStore.containsId(taxonId) && !taxonStore.containsId(taxonId)) {
276 // taxonNamesStore.put(taxonId, zooName);
277 // taxonStore.put(taxonId, taxonBase);
278 // fauEuTaxonMap.put(taxonId, fauEuTaxon);
280 // logger.warn("Ignoring taxon with duplicated id " + taxonId);
283 } catch (Exception e
) {
284 logger
.warn("An exception occurred when creating taxon base with id " + taxonId
+
285 ". Taxon base could not be saved.");
288 } catch (SQLException e
) {
289 logger
.error("SQLException:" + e
);
297 private boolean processTaxaSecondPass(FaunaEuropaeaImportConfigurator fauEuConfig
,
298 Map
<String
, MapWrapper
<?
extends CdmBase
>> stores
,
299 Map
<Integer
, FaunaEuropaeaTaxon
> fauEuTaxonMap
) {
301 if(logger
.isInfoEnabled()) { logger
.info("Processing taxa second pass..."); }
303 MapWrapper
<TaxonBase
> taxonStore
= (MapWrapper
<TaxonBase
>)stores
.get(ICdmIO
.TAXON_STORE
);
305 boolean success
= true;
307 for (int id
: taxonStore
.keySet())
309 TaxonBase
<?
> taxonBase
= taxonStore
.get(id
);
310 FaunaEuropaeaTaxon fauEuTaxon
= fauEuTaxonMap
.get(id
);
312 // success = updateProperties(fauEuConfig, taxonBase, taxonStore, fauEuTaxon, fauEuTaxonMap);
313 String nameString
= buildTaxonName(fauEuTaxon
, taxonBase
, taxonStore
);
320 /* Build name title cache */
321 private String
buildNameTitleCache(String nameString
, FaunaEuropaeaTaxon fauEuTaxon
) {
323 StringBuilder titleCacheStringBuilder
= new StringBuilder(nameString
);
324 int year
= fauEuTaxon
.getYear();
325 if (year
!= 0) { // TODO: What do do with authors like xp, xf, etc?
326 titleCacheStringBuilder
.append(" ");
327 if (fauEuTaxon
.isParenthesis() == true) {
328 titleCacheStringBuilder
.append("(");
330 titleCacheStringBuilder
.append(fauEuTaxon
.getAuthor());
331 titleCacheStringBuilder
.append(" ");
332 titleCacheStringBuilder
.append(year
);
333 if (fauEuTaxon
.isParenthesis() == true) {
334 titleCacheStringBuilder
.append(")");
337 return titleCacheStringBuilder
.toString();
341 /* Build taxon name */
342 private String
buildTaxonName(FaunaEuropaeaTaxon fauEuTaxon
,
343 TaxonBase
<?
> taxonBase
, MapWrapper
<TaxonBase
> taxonStore
) {
345 String localString
= "";
346 String parentString
= "";
348 FaunaEuropaeaTaxon parent
= null;
349 TaxonNameBase
<?
,?
> parentName
= null;
350 TaxonBase
<?
> parentTaxonBase
= null;
351 String parentNameCache
= null;
352 boolean parentComplete
= false;
354 TaxonNameBase
<?
,?
> taxonName
= taxonBase
.getName();
355 ZoologicalName zooName
= (ZoologicalName
)taxonName
;
357 if (zooName
!= null) {
358 localString
= zooName
.getNameCache();
361 int rank
= fauEuTaxon
.getRankId();
362 if (rank
> R_GENUS
) {
363 StringBuilder parentStringBuilder
= new StringBuilder();
364 if(logger
.isDebugEnabled()) {
365 logger
.debug("Local taxon name: (rank = " + rank
+ ") " + localString
);
368 // The scientific name in FaunaEuropaeaTaxon is set only once it has been built completely,
369 // including parent(s) parts.
371 int parentId
= fauEuTaxon
.getParentId();
372 parent
= fauEuTaxonMap
.get(parentId
);
373 if (parent
!= null) {
374 UUID parentUuid
= parent
.getUuid();
375 if (parentUuid
!= null) {
376 parentTaxonBase
= taxonStore
.get(parentId
);
377 parentNameCache
= parent
.getScientificName();
378 if (parentNameCache
== null) { // parent name has not been built yet
379 if (parentTaxonBase
!= null) {
380 parentName
= parentTaxonBase
.getName();
381 if(parentName
!= null) {
382 parentNameCache
= ((ZoologicalName
)parentName
).getNameCache();
384 logger
.warn("Parent taxon name of taxon (uuid= " + parentUuid
.toString() + "), id = " +
385 parent
.getId() + ") is null");
388 logger
.warn("Parent taxon (uuid= " + parentUuid
.toString() + "), id = " +
389 parent
.getId() + ") is null");
391 if (parent
.getRankId() == R_SUBGENUS
) {
392 parentStringBuilder
.append("(");
394 parentStringBuilder
.append(parentNameCache
);
395 if (parent
.getRankId() == R_SUBGENUS
) {
396 parentStringBuilder
.append(")");
398 // parentStringBuilder.append(" ");
399 // parentStringBuilder.append(((ZoologicalName)taxonName).getNameCache());
400 parentString
= parentStringBuilder
.toString();
401 logger
.info("Parent name part built: " + parentString
);
403 parentComplete
= true;
404 parentString
= parent
.getScientificName();
405 logger
.info("Parent name is complete: " + parentString
);
408 logger
.warn("Parent uuid of " + localString
+ " is null");
411 logger
.warn("Parent of " + localString
+ " is null");
413 if (parent
!= null && parent
.getRankId() > R_GENUS
&& parentComplete
== false) {
414 parentString
= buildTaxonName(parent
, parentTaxonBase
, taxonStore
);
417 StringBuilder concatStringBuilder
= new StringBuilder(parentString
);
418 // if (!concatStringBuilder.equals("")) { concatStringBuilder.append(" "); }
419 concatStringBuilder
.append(" ");
420 concatStringBuilder
.append(localString
);
421 String concatString
= concatStringBuilder
.toString();
422 concatString
= (String
) CdmUtils
.removeDuplicateWhitespace(concatString
.trim());
424 zooName
.setNameCache(concatString
);
425 String titleCache
= buildNameTitleCache(concatString
, fauEuTaxon
);
426 zooName
.setTitleCache(titleCache
);
427 zooName
.setFullTitleCache(titleCache
); // TODO: Add reference, NC status
429 fauEuTaxon
.setScientificName(concatString
);
434 private boolean saveTaxa(Map
<String
, MapWrapper
<?
extends CdmBase
>> stores
) {
436 // MapWrapper<TaxonNameBase<?,?>> taxonNameStore = (MapWrapper<TaxonNameBase<?,?>>)stores.get(ICdmIO.TAXONNAME_STORE);
437 MapWrapper
<TaxonBase
> taxonStore
= (MapWrapper
<TaxonBase
>)stores
.get(ICdmIO
.TAXON_STORE
);
440 int nbrOfTaxa
= highestTaxonIndex
;
441 // int nbrOfTaxa = taxonStore.size();
442 boolean success
= true;
444 if(logger
.isInfoEnabled()) { logger
.info("Saving taxa ..."); }
446 if (nbrOfTaxa
< limit
) { // TODO: test with critical values
449 n
= nbrOfTaxa
/ limit
;
452 if(logger
.isInfoEnabled()) {
453 logger
.info("number of taxa = " + taxonStore
.size()
454 + ", highest taxon index = " + highestTaxonIndex
455 + ", limit = " + limit
459 // save taxa in chunks
461 for (int j
= 1; j
<= n
+ 1; j
++)
464 int start
= offset
* limit
;
466 if(logger
.isInfoEnabled()) { logger
.info("Saving taxa: " + start
+ " - " + (start
+ limit
- 1)); }
468 if(logger
.isInfoEnabled()) {
469 logger
.info("index = " + j
470 + ", offset = " + offset
471 + ", start = " + start
);
475 limit
= nbrOfTaxa
- n
* limit
;
476 if(logger
.isInfoEnabled()) { logger
.info(", n = " + n
+ " limit = " + limit
); }
479 Collection
<TaxonBase
> taxonMapPart
= taxonStore
.objects(start
, limit
);
480 getTaxonService().saveTaxonAll(taxonMapPart
);
482 taxonStore
.removeObjects(start
, limit
);