/**\r
* Copyright (C) 2007 EDIT\r
-* European Distributed Institute of Taxonomy \r
+* European Distributed Institute of Taxonomy\r
* http://www.e-taxonomy.eu\r
-* \r
+*\r
* The contents of this file are subject to the Mozilla Public License Version 1.1\r
* See LICENSE.TXT at the top of this package for the full license terms.\r
*/\r
\r
import java.lang.reflect.Method;\r
import java.sql.ResultSet;\r
-import java.sql.ResultSetMetaData;\r
import java.sql.SQLException;\r
import java.sql.Timestamp;\r
-import java.util.HashMap;\r
import java.util.HashSet;\r
-import java.util.Map;\r
import java.util.Set;\r
import java.util.UUID;\r
\r
+import org.apache.commons.lang.StringUtils;\r
import org.apache.log4j.Logger;\r
+import org.hibernate.NonUniqueObjectException;\r
import org.joda.time.DateTime;\r
\r
-import eu.etaxonomy.cdm.common.CdmUtils;\r
import eu.etaxonomy.cdm.io.common.CdmImportBase;\r
import eu.etaxonomy.cdm.io.common.ICdmIO;\r
+import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;\r
import eu.etaxonomy.cdm.io.common.IPartitionedIO;\r
import eu.etaxonomy.cdm.io.common.ImportHelper;\r
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;\r
import eu.etaxonomy.cdm.io.common.Source;\r
-import eu.etaxonomy.cdm.io.common.IImportConfigurator.EDITOR;\r
import eu.etaxonomy.cdm.io.common.mapping.DbImportMapping;\r
+import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;\r
+import eu.etaxonomy.cdm.model.agent.INomenclaturalAuthor;\r
+import eu.etaxonomy.cdm.model.agent.Person;\r
+import eu.etaxonomy.cdm.model.agent.Team;\r
+import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;\r
import eu.etaxonomy.cdm.model.common.AnnotatableEntity;\r
import eu.etaxonomy.cdm.model.common.Annotation;\r
import eu.etaxonomy.cdm.model.common.AnnotationType;\r
import eu.etaxonomy.cdm.model.common.CdmBase;\r
-import eu.etaxonomy.cdm.model.common.ExtensionType;\r
import eu.etaxonomy.cdm.model.common.IdentifiableEntity;\r
import eu.etaxonomy.cdm.model.common.Language;\r
-import eu.etaxonomy.cdm.model.common.MarkerType;\r
import eu.etaxonomy.cdm.model.common.User;\r
+import eu.etaxonomy.cdm.model.location.Country;\r
+import eu.etaxonomy.cdm.model.location.NamedArea;\r
+import eu.etaxonomy.cdm.model.name.ZoologicalName;\r
+import eu.etaxonomy.cdm.strategy.exceptions.StringNotParsableException;\r
+import eu.etaxonomy.cdm.strategy.parser.INonViralNameParser;\r
+import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;\r
\r
/**\r
* @author a.mueller\r
* @created 20.03.2008\r
- * @version 1.0\r
*/\r
public abstract class GlobisImportBase<CDM_BASE extends CdmBase> extends CdmImportBase<GlobisImportConfigurator, GlobisImportState> implements ICdmIO<GlobisImportState>, IPartitionedIO<GlobisImportState> {\r
private static final Logger logger = Logger.getLogger(GlobisImportBase.class);\r
- \r
+\r
public static final UUID ID_IN_SOURCE_EXT_UUID = UUID.fromString("23dac094-e793-40a4-bad9-649fc4fcfd44");\r
- \r
+\r
//NAMESPACES\r
- \r
- protected static final String AREA_NAMESPACE = "gu";\r
- protected static final String DR_NAMESPACE = "dr";\r
- protected static final String IMAGE_NAMESPACE = "Images";\r
- protected static final String LINKS_NAMESPACE = "Links";\r
- protected static final String NOTES_NAMESPACE = "Notes";\r
- protected static final String LANGUAGE_NAMESPACE = "Language";\r
- protected static final String REFERENCE_NAMESPACE = "Source";\r
- protected static final String SOURCEUSE_NAMESPACE = "tu_sources";\r
- protected static final String TAXON_NAMESPACE = "Taxon";\r
- protected static final String NAME_NAMESPACE = "TaxonName";\r
- protected static final String VERNACULAR_NAMESPACE = "Vernaculars";\r
- protected static final String FEATURE_NAMESPACE = "note.type";\r
- protected static final String EXTENSION_TYPE_NAMESPACE = "ExtensionType";\r
- \r
- \r
-\r
- private String pluralString;\r
- private String dbTableName;\r
- //TODO needed?\r
- private Class cdmTargetClass;\r
-\r
- \r
- \r
+\r
+ protected static final String REFERENCE_NAMESPACE = "Literatur";\r
+ protected static final String TAXON_NAMESPACE = "current_species";\r
+ protected static final String COLLECTION_NAMESPACE = "Collection";\r
+ protected static final String IMAGE_NAMESPACE = "Einzelbilder";\r
+ protected static final String SPEC_TAX_NAMESPACE = "specTax";\r
+ protected static final String TYPE_NAMESPACE = "specTax.SpecTypeDepository";\r
+\r
+ private final String pluralString;\r
+ private final String dbTableName;\r
+ private final Class cdmTargetClass;\r
+\r
+ private final INonViralNameParser<?> parser = NonViralNameParserImpl.NewInstance();\r
+\r
+\r
/**\r
* @param dbTableName\r
- * @param dbTableName2 \r
+ * @param dbTableName2\r
*/\r
- public GlobisImportBase(String pluralString, String dbTableName, Class cdmTargetClass) {\r
+ public GlobisImportBase(String pluralString, String dbTableName, Class<?> cdmTargetClass) {\r
this.pluralString = pluralString;\r
this.dbTableName = dbTableName;\r
this.cdmTargetClass = cdmTargetClass;\r
}\r
\r
- protected boolean doInvoke(GlobisImportState state){\r
+ @Override\r
+ protected void doInvoke(GlobisImportState state){\r
logger.info("start make " + getPluralString() + " ...");\r
- boolean success = true ;\r
GlobisImportConfigurator config = state.getConfig();\r
Source source = config.getSource();\r
- \r
+\r
String strIdQuery = getIdQuery();\r
String strRecordQuery = getRecordQuery(config);\r
\r
int recordsPerTransaction = config.getRecordsPerTransaction();\r
try{\r
- ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery, strRecordQuery, recordsPerTransaction);\r
+ ResultSetPartitioner<GlobisImportState> partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery, strRecordQuery, recordsPerTransaction);\r
while (partitioner.nextPartition()){\r
partitioner.doPartition(this, state);\r
}\r
} catch (SQLException e) {\r
logger.error("SQLException:" + e);\r
- return false;\r
+ state.setUnsuccessfull();\r
}\r
- \r
- logger.info("end make " + getPluralString() + " ... " + getSuccessString(success));\r
- return success;\r
+\r
+ logger.info("end make " + getPluralString() + " ... " + getSuccessString(true));\r
+ return;\r
+ }\r
+\r
+ /**\r
+ * @param authorAndYear\r
+ * @param zooName\r
+ */\r
+ protected void handleAuthorAndYear(String authorAndYear, ZoologicalName zooName, Integer id, GlobisImportState state) {\r
+ if (isBlank(authorAndYear)){\r
+ return;\r
+ }else if ("[Denis & Schifferm\u00FCller], 1775".equals(authorAndYear)){\r
+ handleDenisSchiffermueller(zooName, state);\r
+ return;\r
+ }else{\r
+ try {\r
+ String doubtfulAuthorAndYear = null;\r
+ if(authorAndYear.matches(".+\\,\\s\\[\\d{4}\\].*")){\r
+ doubtfulAuthorAndYear = authorAndYear;\r
+ authorAndYear = authorAndYear.replace("[", "").replace("]", "");\r
+ }\r
+\r
+ parser.parseAuthors(zooName, authorAndYear);\r
+ deduplicateAuthors(zooName, state);\r
+\r
+ if (doubtfulAuthorAndYear != null){\r
+ zooName.setAuthorshipCache(doubtfulAuthorAndYear, true);\r
+ }\r
+\r
+ } catch (StringNotParsableException e) {\r
+ logger.warn("Author could not be parsed: " + authorAndYear + " for id " +id);\r
+ zooName.setAuthorshipCache(authorAndYear, true);\r
+ }\r
+ }\r
+ }\r
+\r
+ /**\r
+ * @param zooName\r
+ * @param state\r
+ */\r
+ private void handleDenisSchiffermueller(ZoologicalName zooName,\r
+ GlobisImportState state) {\r
+ String teamStr = "Denis & Schifferm\u00FCller";\r
+ Team team = state.getTeam(teamStr);\r
+ if (team == null){\r
+ team = Team.NewInstance();\r
+ state.putTeam(teamStr, team);\r
+ getAgentService().save(team);\r
+ }\r
+ zooName.setCombinationAuthorship(team);\r
+ zooName.setPublicationYear(1775);\r
+ zooName.setAuthorshipCache("[Denis & Schifferm\u00FCller], 1775", true);\r
+ }\r
+\r
+\r
+ private void deduplicateAuthors(ZoologicalName zooName, GlobisImportState state) {\r
+ zooName.setCombinationAuthorship(getExistingAuthor(zooName.getCombinationAuthorship(), state));\r
+ zooName.setExCombinationAuthorship(getExistingAuthor(zooName.getExCombinationAuthorship(), state));\r
+ zooName.setBasionymAuthorship(getExistingAuthor(zooName.getBasionymAuthorship(), state));\r
+ zooName.setExBasionymAuthorship(getExistingAuthor(zooName.getExBasionymAuthorship(), state));\r
+ }\r
+\r
+ private TeamOrPersonBase<?> getExistingAuthor(INomenclaturalAuthor nomAuthor, GlobisImportState state) {\r
+ TeamOrPersonBase<?> author = (TeamOrPersonBase<?>)nomAuthor;\r
+ if (author == null){\r
+ return null;\r
+ }\r
+ if (author instanceof Person){\r
+ Person person = state.getPerson(author.getTitleCache());\r
+ return saveAndDecide(person, author, author.getTitleCache(), state);\r
+ }else if (author instanceof Team){\r
+ String key = GlobisAuthorImport.makeTeamKey((Team)author, state, getAgentService());\r
+ Team existingTeam = state.getTeam(key);\r
+ if (existingTeam == null){\r
+ Team newTeam = Team.NewInstance();\r
+ for (Person member :((Team) author).getTeamMembers()){\r
+ Person existingPerson = state.getPerson(member.getTitleCache());\r
+ if (existingPerson != null){\r
+ try {\r
+ getAgentService().update(existingPerson);\r
+ } catch (NonUniqueObjectException nuoe){\r
+ // person already exists in\r
+ existingPerson = CdmBase.deproxy(getAgentService().find(existingPerson.getUuid()), Person.class);\r
+ state.putPerson(existingPerson.getTitleCache(), existingPerson);\r
+ } catch (Exception e) {\r
+ throw new RuntimeException (e);\r
+ }\r
+ newTeam.addTeamMember(existingPerson);\r
+//\r
+// logger.warn("newTeam " + newTeam.getId());\r
+ }else{\r
+ newTeam.addTeamMember(member);\r
+ }\r
+ }\r
+ author = newTeam;\r
+ }\r
+\r
+ return saveAndDecide(existingTeam, author, key, state);\r
+ }else{\r
+ logger.warn("Author type not supported: " + author.getClass().getName());\r
+ return author;\r
+ }\r
+ }\r
+\r
+ private TeamOrPersonBase<?> saveAndDecide(TeamOrPersonBase<?> existing, TeamOrPersonBase<?> author, String key, GlobisImportState state) {\r
+ if (existing != null){\r
+ try {\r
+ getAgentService().update(existing);\r
+ } catch (NonUniqueObjectException nuoe){\r
+ // person already exists in\r
+ existing = CdmBase.deproxy(getAgentService().find(existing.getUuid()), TeamOrPersonBase.class);\r
+ putAgent(existing, key, state);\r
+ } catch (Exception e) {\r
+ throw new RuntimeException (e);\r
+ }\r
+ return existing;\r
+ }else{\r
+ getAgentService().save(author);\r
+ putAgent(author, key, state);\r
+ return author;\r
+ }\r
+ }\r
+\r
+ /**\r
+ * @param author\r
+ * @param key\r
+ * @param state\r
+ */\r
+ private void putAgent(TeamOrPersonBase<?> agent, String key, GlobisImportState state) {\r
+ if (agent instanceof Team){\r
+ state.putTeam(key, (Team)agent);\r
+ }else{\r
+ state.putPerson(key, (Person)agent);\r
+ }\r
+ }\r
+\r
+ /**\r
+ * @param state\r
+ * @param countryStr\r
+ * @return\r
+ */\r
+ protected NamedArea getCountry(GlobisImportState state, String countryStr) {\r
+ NamedArea country = Country.getCountryByLabel(countryStr);\r
+ if (country == null){\r
+ try {\r
+ country = state.getTransformer().getNamedAreaByKey(countryStr);\r
+ } catch (UndefinedTransformerMethodException e) {\r
+ e.printStackTrace();\r
+ }\r
+ }\r
+ return country;\r
}\r
- \r
- public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) {\r
+\r
+\r
+\r
+ @Override\r
+ public boolean doPartition(ResultSetPartitioner partitioner, GlobisImportState state) {\r
boolean success = true ;\r
Set objectsToSave = new HashSet();\r
- \r
+\r
DbImportMapping<?, ?> mapping = getMapping();\r
mapping.initialize(state, cdmTargetClass);\r
- \r
+\r
ResultSet rs = partitioner.getResultSet();\r
try{\r
while (rs.next()){\r
logger.error("SQLException:" + e);\r
return false;\r
}\r
- \r
+\r
partitioner.startDoSave();\r
getCommonService().save(objectsToSave);\r
return success;\r
}\r
\r
\r
- \r
+\r
/**\r
* @return\r
*/\r
- protected abstract DbImportMapping<?, ?> getMapping();\r
- \r
+ protected /*abstract*/ DbImportMapping<?, ?> getMapping(){\r
+ return null;\r
+ }\r
+\r
/**\r
* @return\r
*/\r
String result = " SELECT id FROM " + getTableName();\r
return result;\r
}\r
- \r
+\r
/* (non-Javadoc)\r
* @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getPluralString()\r
*/\r
- public String getPluralString(){\r
+ @Override\r
+ public String getPluralString(){\r
return pluralString;\r
}\r
\r
protected String getTableName(){\r
return this.dbTableName;\r
}\r
- \r
+\r
protected boolean doIdCreatedUpdatedNotes(GlobisImportState state, IdentifiableEntity identifiableEntity, ResultSet rs, long id, String namespace)\r
throws SQLException{\r
boolean success = true;\r
//id\r
- success &= ImportHelper.setOriginalSource(identifiableEntity, state.getConfig().getSourceReference(), id, namespace);\r
+ success &= ImportHelper.setOriginalSource(identifiableEntity, state.getTransactionalSourceReference(), id, namespace);\r
//createdUpdateNotes\r
success &= doCreatedUpdatedNotes(state, identifiableEntity, rs, namespace);\r
return success;\r
}\r
- \r
- \r
+\r
+\r
protected boolean doCreatedUpdatedNotes(GlobisImportState state, AnnotatableEntity annotatableEntity, ResultSet rs, String namespace)\r
throws SQLException{\r
\r
GlobisImportConfigurator config = state.getConfig();\r
Object createdWhen = rs.getObject("Created_When");\r
String createdWho = rs.getString("Created_Who");\r
- Object updatedWhen = null;\r
- String updatedWho = null;\r
- try {\r
- updatedWhen = rs.getObject("Updated_When");\r
- updatedWho = rs.getString("Updated_who");\r
- } catch (SQLException e) {\r
- //Table "Name" has no updated when/who\r
- }\r
+ Object updatedWhen = rs.getObject("Updated_When");\r
+ String updatedWho = rs.getString("Updated_who");\r
String notes = rs.getString("notes");\r
- \r
+\r
boolean success = true;\r
- \r
+\r
//Created When, Who, Updated When Who\r
if (config.getEditor() == null || config.getEditor().equals(EDITOR.NO_EDITORS)){\r
//do nothing\r
}else {\r
logger.warn("Editor type not yet implemented: " + config.getEditor());\r
}\r
- \r
- \r
+\r
+\r
//notes\r
- if (CdmUtils.isNotEmpty(notes)){\r
+ if (StringUtils.isNotBlank(notes)){\r
String notesString = String.valueOf(notes);\r
if (notesString.length() > 65530 ){\r
notesString = notesString.substring(0, 65530) + "...";\r
}\r
return success;\r
}\r
- \r
+\r
private User getUser(String userString, GlobisImportState state){\r
- if (CdmUtils.isEmpty(userString)){\r
+ if (isBlank(userString)){\r
return null;\r
}\r
userString = userString.trim();\r
- \r
+\r
User user = state.getUser(userString);\r
if (user == null){\r
user = getTransformedUser(userString,state);\r
}\r
return user;\r
}\r
- \r
+\r
private User getTransformedUser(String userString, GlobisImportState state){\r
Method method = state.getConfig().getUserTransformationMethod();\r
if (method == null){\r
}\r
\r
private User makeNewUser(String userString, GlobisImportState state){\r
- String pwd = getPassword(); \r
+ String pwd = getPassword();\r
User user = User.NewInstance(userString, pwd);\r
state.putUser(userString, user);\r
getUserService().save(user);\r
logger.info("Added new user: " + userString);\r
return user;\r
}\r
- \r
+\r
private String getPassword(){\r
String result = UUID.randomUUID().toString();\r
return result;\r
}\r
- \r
+\r
private DateTime getDateTime(Object timeString){\r
if (timeString == null){\r
return null;\r
}\r
return dateTime;\r
}\r
- \r
- protected boolean resultSetHasColumn(ResultSet rs, String columnName){\r
- try {\r
- ResultSetMetaData metaData = rs.getMetaData();\r
- for (int i = 0; i < metaData.getColumnCount(); i++){\r
- if (metaData.getColumnName(i + 1).equalsIgnoreCase(columnName)){\r
- return true;\r
- }\r
- }\r
- return false;\r
- } catch (SQLException e) {\r
- logger.warn("Exception in resultSetHasColumn");\r
- return false;\r
- }\r
- }\r
- \r
- protected boolean checkSqlServerColumnExists(Source source, String tableName, String columnName){\r
- String strQuery = "SELECT Count(t.id) as n " +\r
- " FROM sysobjects AS t " +\r
- " INNER JOIN syscolumns AS c ON t.id = c.id " +\r
- " WHERE (t.xtype = 'U') AND " + \r
- " (t.name = '" + tableName + "') AND " + \r
- " (c.name = '" + columnName + "')";\r
- ResultSet rs = source.getResultSet(strQuery) ; \r
- int n;\r
- try {\r
- rs.next();\r
- n = rs.getInt("n");\r
- return n>0;\r
- } catch (SQLException e) {\r
- e.printStackTrace();\r
- return false;\r
- }\r
- \r
- }\r
- \r
- /**\r
- * Returns a map that holds all values of a ResultSet. This is needed if a value needs to\r
- * be accessed twice\r
- * @param rs\r
- * @return\r
- * @throws SQLException\r
- */\r
- protected Map<String, Object> getValueMap(ResultSet rs) throws SQLException{\r
- try{\r
- Map<String, Object> valueMap = new HashMap<String, Object>();\r
- int colCount = rs.getMetaData().getColumnCount();\r
- for (int c = 0; c < colCount ; c++){\r
- Object value = rs.getObject(c+1);\r
- String label = rs.getMetaData().getColumnLabel(c+1).toLowerCase();\r
- if (value != null && ! CdmUtils.Nz(value.toString()).trim().equals("")){\r
- valueMap.put(label, value);\r
- }\r
- }\r
- return valueMap;\r
- }catch(SQLException e){\r
- throw e;\r
- }\r
- }\r
- \r
- protected ExtensionType getExtensionType(UUID uuid, String label, String text, String labelAbbrev){\r
- ExtensionType extensionType = (ExtensionType)getTermService().find(uuid);\r
- if (extensionType == null){\r
- extensionType = ExtensionType.NewInstance(text, label, labelAbbrev);\r
- extensionType.setUuid(uuid);\r
- getTermService().save(extensionType);\r
- }\r
- return extensionType;\r
- }\r
- \r
- protected MarkerType getMarkerType(UUID uuid, String label, String text, String labelAbbrev){\r
- MarkerType markerType = (MarkerType)getTermService().find(uuid);\r
- if (markerType == null){\r
- markerType = MarkerType.NewInstance(label, text, labelAbbrev);\r
- markerType.setUuid(uuid);\r
- getTermService().save(markerType);\r
- }\r
- return markerType;\r
- }\r
- \r
+\r
+\r
\r
/**\r
* Reads a foreign key field from the result set and adds its value to the idSet.\r
idSet.add(id);\r
}\r
}\r
- \r
+\r
+\r
+\r
+\r
/**\r
* Returns true if i is a multiple of recordsPerTransaction\r
* @param i\r
startTransaction();\r
return (i % recordsPerLoop) == 0;\r
}\r
- \r
+\r
protected void doLogPerLoop(int count, int recordsPerLog, String pluralString){\r
if ((count % recordsPerLog ) == 0 && count!= 0 ){ logger.info(pluralString + " handled: " + (count));}\r
}\r
- \r
\r
\r
- \r
+\r
+\r
}\r
BGBM BDI Projects / cdmlib-apps.git / blobdiff