Revision 0c8759ce
Added by U-BGBM\k.luther over 8 years ago
| cdm-pesi/src/main/java/eu/etaxonomy/cdm/app/common/CdmDestinations.java | ||
|---|---|---|
| 34 | 34 |
public static ICdmDataSource cdm_test_local_faunaEu_mysql(){
|
| 35 | 35 |
DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL; |
| 36 | 36 |
String cdmServer = "127.0.0.1"; |
| 37 |
String cdmDB = "cdm_faunaeu_2015_11";
|
|
| 37 |
String cdmDB = "cdm_faunaeu_2015_09";
|
|
| 38 | 38 |
String cdmUserName = "root"; |
| 39 | 39 |
return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null); |
| 40 | 40 |
} |
| cdm-pesi/src/main/java/eu/etaxonomy/cdm/app/pesi/FaunaEuropaeaActivator.java | ||
|---|---|---|
| 17 | 17 |
import eu.etaxonomy.cdm.app.common.CdmDestinations; |
| 18 | 18 |
import eu.etaxonomy.cdm.database.DbSchemaValidation; |
| 19 | 19 |
import eu.etaxonomy.cdm.database.ICdmDataSource; |
| 20 |
import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper; |
|
| 21 | 20 |
import eu.etaxonomy.cdm.io.common.CdmDefaultImport; |
| 22 | 21 |
import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK; |
| 23 | 22 |
import eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES; |
| ... | ... | |
| 47 | 46 |
|
| 48 | 47 |
// static final CHECK check = CHECK.CHECK_AND_IMPORT; |
| 49 | 48 |
static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK; |
| 50 |
// static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.CREATE;
|
|
| 49 |
static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.CREATE; |
|
| 51 | 50 |
// static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.UPDATE; |
| 52 |
static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.VALIDATE; |
|
| 51 |
// static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.VALIDATE; |
|
| 52 |
|
|
| 53 | 53 |
|
| 54 | 54 |
|
| 55 |
|
|
| 56 | 55 |
static final NomenclaturalCode nomenclaturalCode = NomenclaturalCode.ICZN; |
| 57 | 56 |
|
| 58 | 57 |
// ****************** ALL ***************************************** |
| 59 | 58 |
|
| 60 | 59 |
// Fauna Europaea to CDM import |
| 61 | 60 |
static final boolean doAuthors = false; |
| 62 |
static final boolean doTaxa = false;
|
|
| 61 |
static final boolean doTaxa =false; |
|
| 63 | 62 |
static final boolean doBasionyms = false; |
| 64 |
static final boolean doTaxonomicallyIncluded = false;
|
|
| 63 |
static final boolean doTaxonomicallyIncluded = true;
|
|
| 65 | 64 |
static final boolean doMisappliedNames = false; |
| 66 | 65 |
static final boolean doHeterotypicSynonyms = false; |
| 67 |
static final DO_REFERENCES doReferences = DO_REFERENCES.ALL;
|
|
| 66 |
static final DO_REFERENCES doReferences = DO_REFERENCES.NONE;
|
|
| 68 | 67 |
static final boolean doDistributions = false; |
| 69 | 68 |
static final boolean makeFeatureTree = false; |
| 70 |
static final boolean doVernacularNames = true;
|
|
| 71 |
static final boolean doAssociatedSpecialists = false;
|
|
| 69 |
static final boolean doVernacularNames = false;
|
|
| 70 |
static final boolean doAssociatedSpecialists = true;
|
|
| 72 | 71 |
static final boolean addCommonNameFeature = false; |
| 72 |
static final boolean doInferredSynonyms = false; |
|
| 73 | 73 |
// CDM to CDM import |
| 74 | 74 |
static final boolean doHeterotypicSynonymsForBasionyms = false; |
| 75 |
|
|
| 75 |
|
|
| 76 | 76 |
// ************************ NONE **************************************** // |
| 77 | 77 |
|
| 78 | 78 |
// Fauna Europaea to CDM import |
| ... | ... | |
| 118 | 118 |
fauEuImportConfigurator.setSourceRefUuid(PesiTransformer.uuidSourceRefFaunaEuropaea); |
| 119 | 119 |
fauEuImportConfigurator.setDoAssociatedSpecialists(doAssociatedSpecialists); |
| 120 | 120 |
fauEuImportConfigurator.setDoVernacularNames(doVernacularNames); |
| 121 |
fauEuImportConfigurator.setDoInferredSynonyms(doInferredSynonyms); |
|
| 121 | 122 |
CdmDefaultImport<FaunaEuropaeaImportConfigurator> fauEuImport = |
| 122 | 123 |
new CdmDefaultImport<FaunaEuropaeaImportConfigurator>(); |
| 123 | 124 |
try {
|
| ... | ... | |
| 171 | 172 |
FeatureNode commonNameFeatureNode = FeatureNode.NewInstance(commonNameFeature); |
| 172 | 173 |
app.getFeatureTreeService().saveOrUpdate(featureTree); |
| 173 | 174 |
} |
| 174 |
|
|
| 175 |
|
|
| 175 | 176 |
System.out.println("End importing Fauna Europaea data");
|
| 176 | 177 |
} |
| 177 | 178 |
|
| cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaImportBase.java | ||
|---|---|---|
| 25 | 25 |
import eu.etaxonomy.cdm.io.common.CdmImportBase; |
| 26 | 26 |
import eu.etaxonomy.cdm.io.common.ICdmImport; |
| 27 | 27 |
import eu.etaxonomy.cdm.io.common.Source; |
| 28 |
import eu.etaxonomy.cdm.io.pesi.out.PesiTransformer; |
|
| 29 |
import eu.etaxonomy.cdm.model.common.MarkerType; |
|
| 28 | 30 |
import eu.etaxonomy.cdm.model.reference.Reference; |
| 29 | 31 |
import eu.etaxonomy.cdm.model.taxon.Classification; |
| 30 | 32 |
|
| ... | ... | |
| 167 | 169 |
return matcher; |
| 168 | 170 |
} |
| 169 | 171 |
|
| 172 |
protected MarkerType getUuidMarkerType(UUID uuid, FaunaEuropaeaImportState state){
|
|
| 173 |
if (uuid == null){
|
|
| 174 |
uuid = UUID.randomUUID(); |
|
| 175 |
} |
|
| 170 | 176 |
|
| 177 |
MarkerType markerType = state.getMarkerType(uuid); |
|
| 178 |
if (markerType == null){
|
|
| 179 |
markerType = (MarkerType)getTermService().find(uuid); |
|
| 180 |
if (markerType == null){
|
|
| 181 |
if (uuid.equals(PesiTransformer.uuidMarkerGuidIsMissing)){
|
|
| 182 |
markerType = MarkerType.NewInstance("Uuid is Missing", "Uuid is missing", null);
|
|
| 183 |
markerType.setUuid(uuid); |
|
| 184 |
} else if (uuid.equals(PesiTransformer.uuidMarkerTypeHasNoLastAction)){
|
|
| 185 |
markerType = MarkerType.NewInstance("Has no last Action", "Has no last action", null);
|
|
| 186 |
markerType.setUuid(uuid); |
|
| 187 |
} |
|
| 188 |
markerType = (MarkerType)getTermService().save(markerType); |
|
| 189 |
} |
|
| 190 |
} |
|
| 191 |
|
|
| 192 |
state.putMarkerType(markerType); |
|
| 193 |
return markerType; |
|
| 194 |
} |
|
| 171 | 195 |
|
| 172 | 196 |
} |
| cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaImportConfigurator.java | ||
|---|---|---|
| 39 | 39 |
private boolean doOccurrence = true; |
| 40 | 40 |
private boolean doVernacularNames = true; |
| 41 | 41 |
private boolean doAssociatedSpecialists = true; |
| 42 |
private boolean doInferredSynonyms = true; |
|
| 42 | 43 |
|
| 43 | 44 |
public boolean isDoVernacularNames() {
|
| 44 | 45 |
return doVernacularNames; |
| ... | ... | |
| 52 | 53 |
return doTypes; |
| 53 | 54 |
} |
| 54 | 55 |
|
| 55 |
|
|
| 56 |
|
|
| 56 | 57 |
|
| 57 | 58 |
|
| 58 | 59 |
/* Max number of taxa to be saved with one service call */ |
| ... | ... | |
| 106 | 107 |
//TODO |
| 107 | 108 |
if (this.sourceReference == null){
|
| 108 | 109 |
logger.warn("getSource Reference not yet fully implemented");
|
| 110 |
|
|
| 109 | 111 |
sourceReference = ReferenceFactory.newDatabase(); |
| 110 | 112 |
|
| 111 | 113 |
sourceReference.setTitleCache("Fauna Europaea database", true);
|
| ... | ... | |
| 304 | 306 |
} |
| 305 | 307 |
|
| 306 | 308 |
public boolean isDoAssociatedSpecialists() {
|
| 307 |
|
|
| 309 |
|
|
| 308 | 310 |
return this.doAssociatedSpecialists; |
| 309 | 311 |
} |
| 310 |
|
|
| 312 |
|
|
| 311 | 313 |
public void setDoAssociatedSpecialists(boolean doAssociatedSpecialists){
|
| 312 | 314 |
this.doAssociatedSpecialists = doAssociatedSpecialists; |
| 313 | 315 |
} |
| 314 | 316 |
|
| 317 |
/** |
|
| 318 |
* @return the doInferredSynonyms |
|
| 319 |
*/ |
|
| 320 |
public boolean isDoInferredSynonyms() {
|
|
| 321 |
return doInferredSynonyms; |
|
| 322 |
} |
|
| 323 |
|
|
| 324 |
public void setDoInferredSynonyms(boolean doInferredSynonyms){
|
|
| 325 |
this.doInferredSynonyms = doInferredSynonyms; |
|
| 326 |
} |
|
| 315 | 327 |
|
| 316 | 328 |
} |
| cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaRefImport.java | ||
|---|---|---|
| 75 | 75 |
if (state.getConfig().getDoReferences().equals(DO_REFERENCES.NONE)){
|
| 76 | 76 |
return; |
| 77 | 77 |
} |
| 78 |
if (state.getConfig().getSourceReference().getId() == 0){
|
|
| 79 |
Reference sourceRef = getReferenceService().find(state.getConfig().getSourceRefUuid()); |
|
| 80 |
state.getConfig().setSourceReference(sourceRef); |
|
| 81 |
} |
|
| 78 | 82 |
Set<UUID> taxonUuids = null; |
| 79 | 83 |
Map<Integer, Reference<?>> references = null; |
| 80 | 84 |
Map<String,TeamOrPersonBase<?>> authors = null; |
| ... | ... | |
| 175 | 179 |
int refId = rsRefs.getInt("ref_id");
|
| 176 | 180 |
String var = "\u00A7"; |
| 177 | 181 |
String refAuthor = deleteSymbol(var,rsRefs.getString("ref_author"));
|
| 178 |
|
|
| 182 |
|
|
| 179 | 183 |
String year = deleteSymbol(var, rsRefs.getString("ref_year"));
|
| 180 | 184 |
String title = deleteSymbol(var, rsRefs.getString("ref_title"));
|
| 181 | 185 |
|
| ... | ... | |
| 281 | 285 |
private void commitReferences(Map<Integer, Reference<?>> references, |
| 282 | 286 |
Map<String, TeamOrPersonBase<?>> authors, |
| 283 | 287 |
Map<Integer, UUID> referenceUuids, int i, TransactionStatus txStatus) {
|
| 284 |
|
|
| 288 |
|
|
| 285 | 289 |
Map <UUID, Reference> referenceMap =getReferenceService().save((Collection)references.values()); |
| 286 | 290 |
logger.info("i = " + i + " - references saved");
|
| 287 | 291 |
|
| ... | ... | |
| 447 | 451 |
uuid = referenceUuids.get(itRefs.next()); |
| 448 | 452 |
uuidSet.add(uuid); |
| 449 | 453 |
} |
| 450 |
|
|
| 454 |
|
|
| 451 | 455 |
referenceList = getReferenceService().find(uuidSet); |
| 452 |
|
|
| 456 |
|
|
| 453 | 457 |
references = new HashMap<Integer, Reference>(limit); |
| 454 | 458 |
for (Reference<?> ref : referenceList){
|
| 455 | 459 |
references.put(Integer.valueOf(((OriginalSourceBase)ref.getSources().iterator().next()).getIdInSource()), ref); |
| ... | ... | |
| 527 | 531 |
protected boolean isIgnore(FaunaEuropaeaImportState state){
|
| 528 | 532 |
return (state.getConfig().getDoReferences() == IImportConfigurator.DO_REFERENCES.NONE); |
| 529 | 533 |
} |
| 530 |
|
|
| 534 |
|
|
| 531 | 535 |
private String deleteSymbol(String symbol, String stringVar){
|
| 532 | 536 |
if (stringVar.startsWith(symbol)){
|
| 533 | 537 |
if (stringVar.endsWith(symbol)){
|
| cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaRelTaxonIncludeImport.java | ||
|---|---|---|
| 13 | 13 |
|
| 14 | 14 |
import java.sql.ResultSet; |
| 15 | 15 |
import java.sql.SQLException; |
| 16 |
import java.util.ArrayList; |
|
| 16 | 17 |
import java.util.HashMap; |
| 17 | 18 |
import java.util.HashSet; |
| 18 | 19 |
import java.util.List; |
| ... | ... | |
| 29 | 30 |
import eu.etaxonomy.cdm.io.common.ICdmIO; |
| 30 | 31 |
import eu.etaxonomy.cdm.io.common.MapWrapper; |
| 31 | 32 |
import eu.etaxonomy.cdm.io.common.Source; |
| 33 |
import eu.etaxonomy.cdm.io.pesi.out.PesiTransformer; |
|
| 32 | 34 |
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase; |
| 33 | 35 |
import eu.etaxonomy.cdm.model.common.Annotation; |
| 34 | 36 |
import eu.etaxonomy.cdm.model.common.CdmBase; |
| 37 |
import eu.etaxonomy.cdm.model.common.IdentifiableSource; |
|
| 35 | 38 |
import eu.etaxonomy.cdm.model.common.Language; |
| 39 |
import eu.etaxonomy.cdm.model.common.Marker; |
|
| 40 |
import eu.etaxonomy.cdm.model.common.MarkerType; |
|
| 41 |
import eu.etaxonomy.cdm.model.name.Rank; |
|
| 42 |
import eu.etaxonomy.cdm.model.name.TaxonNameBase; |
|
| 43 |
import eu.etaxonomy.cdm.model.name.ZoologicalName; |
|
| 36 | 44 |
import eu.etaxonomy.cdm.model.reference.Reference; |
| 37 | 45 |
import eu.etaxonomy.cdm.model.taxon.Classification; |
| 38 | 46 |
import eu.etaxonomy.cdm.model.taxon.Synonym; |
| 47 |
import eu.etaxonomy.cdm.model.taxon.SynonymRelationship; |
|
| 39 | 48 |
import eu.etaxonomy.cdm.model.taxon.SynonymRelationshipType; |
| 40 | 49 |
import eu.etaxonomy.cdm.model.taxon.Taxon; |
| 41 | 50 |
import eu.etaxonomy.cdm.model.taxon.TaxonBase; |
| ... | ... | |
| 61 | 70 |
|
| 62 | 71 |
|
| 63 | 72 |
private Reference<?> sourceRef; |
| 73 |
private final String parentPluralString = "Taxa"; |
|
| 64 | 74 |
private static String ALL_SYNONYM_FROM_CLAUSE = " FROM Taxon INNER JOIN Taxon AS Parent " + |
| 65 | 75 |
" ON Taxon.TAX_TAX_IDPARENT = Parent.TAX_ID " + |
| 66 | 76 |
" WHERE (Taxon.TAX_VALID = 0) " + |
| ... | ... | |
| 79 | 89 |
@Override |
| 80 | 90 |
protected boolean isIgnore(FaunaEuropaeaImportState state) {
|
| 81 | 91 |
return ! (state.getConfig().isDoTaxonomicallyIncluded() || |
| 82 |
state.getConfig().isDoMisappliedNames() || state.getConfig().isDoHeterotypicSynonyms()); |
|
| 92 |
state.getConfig().isDoMisappliedNames() || state.getConfig().isDoHeterotypicSynonyms() || state.getConfig().isDoInferredSynonyms());
|
|
| 83 | 93 |
} |
| 84 | 94 |
|
| 85 | 95 |
private boolean checkTaxonStatus(FaunaEuropaeaImportConfigurator fauEuConfig) {
|
| ... | ... | |
| 101 | 111 |
/*logger.warn("Start RelTaxon doInvoke");
|
| 102 | 112 |
ProfilerController.memorySnapshot(); |
| 103 | 113 |
*/ |
| 104 |
|
|
| 105 |
if (!state.getConfig().isDoTaxonomicallyIncluded()){
|
|
| 106 |
return; |
|
| 107 |
} |
|
| 114 |
|
|
| 115 |
|
|
| 108 | 116 |
Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores(); |
| 109 | 117 |
|
| 110 | 118 |
MapWrapper<TeamOrPersonBase> authorStore = (MapWrapper<TeamOrPersonBase>)stores.get(ICdmIO.TEAM_STORE); |
| ... | ... | |
| 127 | 135 |
|
| 128 | 136 |
if (state.getConfig().isDoTaxonomicallyIncluded()) {
|
| 129 | 137 |
processParentsChildren(state); |
| 130 |
|
|
| 138 |
|
|
| 131 | 139 |
} |
| 132 | 140 |
if (state.getConfig().isDoAssociatedSpecialists()){
|
| 133 | 141 |
processAssociatedSpecialists(state); |
| ... | ... | |
| 150 | 158 |
} |
| 151 | 159 |
processHeterotypicSynonyms(state, ALL_SYNONYM_FROM_CLAUSE); |
| 152 | 160 |
} |
| 161 |
|
|
| 162 |
if (state.getConfig().isDoInferredSynonyms()){
|
|
| 163 |
processInferredSynonyms(state); |
|
| 164 |
} |
|
| 153 | 165 |
/* |
| 154 | 166 |
logger.warn("End RelTaxon doInvoke");
|
| 155 | 167 |
ProfilerController.memorySnapshot(); |
| ... | ... | |
| 261 | 273 |
if (((i % limit) == 0 && i != 1 ) || i == count ) {
|
| 262 | 274 |
|
| 263 | 275 |
createAndCommitParentChildRelationships( |
| 264 |
state, txStatus, childParentMap, taxonGroupCoordinatorMap, taxonSpecialistMap);
|
|
| 276 |
state, txStatus, childParentMap, taxonSpecialistMap, taxonGroupCoordinatorMap);
|
|
| 265 | 277 |
childParentMap = null; |
| 266 | 278 |
|
| 267 | 279 |
if(logger.isInfoEnabled()) {
|
| ... | ... | |
| 893 | 905 |
|
| 894 | 906 |
return; |
| 895 | 907 |
} |
| 896 |
|
|
| 908 |
|
|
| 897 | 909 |
private void processAssociatedSpecialists(FaunaEuropaeaImportState state){
|
| 898 | 910 |
int limit = state.getConfig().getLimitSave(); |
| 899 | 911 |
|
| ... | ... | |
| 901 | 913 |
|
| 902 | 914 |
Map<UUID, UUID> childParentMap = null; |
| 903 | 915 |
|
| 904 |
|
|
| 916 |
|
|
| 905 | 917 |
TaxonNode taxonNode = null; |
| 906 | 918 |
TeamOrPersonBase associatedSpecialist = null; |
| 907 | 919 |
TaxonNodeAgentRelation agentRel = null; |
| ... | ... | |
| 912 | 924 |
|
| 913 | 925 |
String selectCount = |
| 914 | 926 |
" SELECT count(*) "; |
| 915 |
|
|
| 916 |
|
|
| 917 |
|
|
| 927 |
|
|
| 928 |
|
|
| 929 |
|
|
| 918 | 930 |
String selectColumns = "SELECT u.USR_GROUPNAME as groupName, u.USR_GROUPNOTE groupNote, u.USR_USR_ID as user_user_id, " |
| 919 | 931 |
+ " u.usr_id user_id, user2.USR_ID as user2_id, taxon.UUID as tax_uuid "; |
| 920 |
|
|
| 932 |
|
|
| 921 | 933 |
String fromClause = " FROM USERROLE as userrole left join USERS u on userrole.URL_USR_ID = u.USR_ID left join USERS user2 on user2.USR_ID = u.USR_USR_ID " |
| 922 | 934 |
+ " left join TAXON taxon on (taxon.TAX_USR_IDSP= user2.USR_ID or taxon.TAX_USR_IDGC= user2.USR_ID) " |
| 923 | 935 |
+ " where USERROLE.URL_ROL_ID = 7 order by taxon.TAX_ID"; |
| ... | ... | |
| 928 | 940 |
|
| 929 | 941 |
String selectQuery = |
| 930 | 942 |
selectColumns + fromClause + orderClause; |
| 931 |
|
|
| 943 |
|
|
| 932 | 944 |
if(logger.isInfoEnabled()) { logger.info("Start making associated specialists..."); }
|
| 933 | 945 |
|
| 934 | 946 |
try {
|
| ... | ... | |
| 967 | 979 |
logger.info("There is an associated specialist for a taxon which has no taxonnode.");
|
| 968 | 980 |
} |
| 969 | 981 |
} |
| 970 |
|
|
| 982 |
|
|
| 971 | 983 |
agentUuid = state.getAgentMap().get(userId); |
| 972 |
|
|
| 973 |
|
|
| 984 |
|
|
| 985 |
|
|
| 974 | 986 |
associatedSpecialist = (TeamOrPersonBase) getAgentService().find(agentUuid); |
| 975 | 987 |
if (associatedSpecialist != null && taxonNode != null){
|
| 976 | 988 |
agentRel =taxonNode.addAgentRelation(FaunaEuropaeaTransformer.getAssociateSpecialistType(getTermService()), associatedSpecialist); |
| 977 |
if (!StringUtils.isBlank(groupName)) |
|
| 978 |
agentRel.addAnnotation(Annotation.NewInstance(groupName, Language.DEFAULT())); |
|
| 989 |
if (!StringUtils.isBlank(groupName)) {
|
|
| 990 |
agentRel.addAnnotation(Annotation.NewInstance(groupName, Language.DEFAULT())); |
|
| 991 |
} |
|
| 979 | 992 |
} |
| 980 |
|
|
| 981 |
|
|
| 993 |
|
|
| 994 |
|
|
| 982 | 995 |
} |
| 983 | 996 |
getTaxonNodeService().saveOrUpdate(taxonNode); |
| 984 | 997 |
commitTransaction(txStatus); |
| ... | ... | |
| 986 | 999 |
logger.info("Problems during creating associated specialists.", e);
|
| 987 | 1000 |
} |
| 988 | 1001 |
} |
| 989 |
|
|
| 1002 |
|
|
| 1003 |
private void processInferredSynonyms(FaunaEuropaeaImportState state){
|
|
| 1004 |
|
|
| 1005 |
int count; |
|
| 1006 |
int pastCount; |
|
| 1007 |
boolean success = true; |
|
| 1008 |
// Get the limit for objects to save within a single transaction. |
|
| 1009 |
if (! state.getConfig().isDoInferredSynonyms()){
|
|
| 1010 |
logger.info ("Ignore Creating Inferred Synonyms...");
|
|
| 1011 |
return; |
|
| 1012 |
} |
|
| 1013 |
|
|
| 1014 |
int limit = state.getConfig().getLimitSave(); |
|
| 1015 |
// Create inferred synonyms for accepted taxa |
|
| 1016 |
logger.info("Creating Inferred Synonyms...");
|
|
| 1017 |
|
|
| 1018 |
|
|
| 1019 |
count = 0; |
|
| 1020 |
pastCount = 0; |
|
| 1021 |
int pageSize = limit/10; |
|
| 1022 |
int pageNumber = 1; |
|
| 1023 |
String inferredSynonymPluralString = "Inferred Synonyms"; |
|
| 1024 |
|
|
| 1025 |
// Start transaction |
|
| 1026 |
TransactionStatus txStatus = startTransaction(true); |
|
| 1027 |
logger.info("Started new transaction. Fetching some " + parentPluralString + " first (max: " + limit + ") ...");
|
|
| 1028 |
List<TaxonBase> taxonList = null; |
|
| 1029 |
Set<TaxonBase> synonymList = new HashSet<TaxonBase>(); |
|
| 1030 |
|
|
| 1031 |
|
|
| 1032 |
while ((taxonList = getTaxonService().listTaxaByName(Taxon.class, "*", "*", "*", "*", Rank.SPECIES(), pageSize, pageNumber)).size() > 0) {
|
|
| 1033 |
HashMap<Integer, TaxonNameBase<?,?>> inferredSynonymsDataToBeSaved = new HashMap<Integer, TaxonNameBase<?,?>>(); |
|
| 1034 |
|
|
| 1035 |
logger.info("Fetched " + taxonList.size() + " " + parentPluralString + ". Importing...");
|
|
| 1036 |
synonymList = createInferredSynonymsForTaxonList(state, taxonList); |
|
| 1037 |
getTaxonService().save(synonymList); |
|
| 1038 |
|
|
| 1039 |
// getTaxonService().saveOrUpdate(taxonList); |
|
| 1040 |
// Commit transaction |
|
| 1041 |
commitTransaction(txStatus); |
|
| 1042 |
logger.debug("Committed transaction.");
|
|
| 1043 |
logger.info("Imported " + (taxonList.size()) + " " + inferredSynonymPluralString + ". Total: " + count);
|
|
| 1044 |
//pastCount = count; |
|
| 1045 |
|
|
| 1046 |
|
|
| 1047 |
|
|
| 1048 |
// Start transaction |
|
| 1049 |
txStatus = startTransaction(true); |
|
| 1050 |
logger.info("Started new transaction. Fetching some " + parentPluralString + " first (max: " + limit + ") ...");
|
|
| 1051 |
|
|
| 1052 |
// Increment pageNumber |
|
| 1053 |
pageNumber++; |
|
| 1054 |
} |
|
| 1055 |
taxonList = null; |
|
| 1056 |
while ((taxonList = getTaxonService().listTaxaByName(Taxon.class, "*", "*", "*", "*", Rank.SUBSPECIES(), pageSize, pageNumber)).size() > 0) {
|
|
| 1057 |
HashMap<Integer, TaxonNameBase<?,?>> inferredSynonymsDataToBeSaved = new HashMap<Integer, TaxonNameBase<?,?>>(); |
|
| 1058 |
|
|
| 1059 |
logger.info("Fetched " + taxonList.size() + " " + parentPluralString + ". Exporting...");
|
|
| 1060 |
synonymList = createInferredSynonymsForTaxonList(state, taxonList); |
|
| 1061 |
|
|
| 1062 |
getTaxonService().save(synonymList); |
|
| 1063 |
// Commit transaction |
|
| 1064 |
commitTransaction(txStatus); |
|
| 1065 |
logger.debug("Committed transaction.");
|
|
| 1066 |
logger.info("Exported " + taxonList.size()+ " " + inferredSynonymPluralString + ". Total: " + count);
|
|
| 1067 |
//pastCount = count; |
|
| 1068 |
|
|
| 1069 |
|
|
| 1070 |
|
|
| 1071 |
// Start transaction |
|
| 1072 |
txStatus = startTransaction(true); |
|
| 1073 |
logger.info("Started new transaction. Fetching some " + parentPluralString + " first (max: " + limit + ") ...");
|
|
| 1074 |
|
|
| 1075 |
// Increment pageNumber |
|
| 1076 |
pageNumber++; |
|
| 1077 |
inferredSynonymsDataToBeSaved = null; |
|
| 1078 |
} |
|
| 1079 |
if (taxonList.size() == 0) {
|
|
| 1080 |
logger.info("No " + parentPluralString + " left to fetch.");
|
|
| 1081 |
} |
|
| 1082 |
|
|
| 1083 |
taxonList = null; |
|
| 1084 |
|
|
| 1085 |
// Commit transaction |
|
| 1086 |
commitTransaction(txStatus); |
|
| 1087 |
System.gc(); |
|
| 1088 |
|
|
| 1089 |
//ProfilerController.memorySnapshot(); |
|
| 1090 |
logger.debug("Committed transaction.");
|
|
| 1091 |
|
|
| 1092 |
|
|
| 1093 |
} |
|
| 1094 |
|
|
| 1095 |
/** |
|
| 1096 |
* @param state |
|
| 1097 |
* @param mapping |
|
| 1098 |
* @param synRelMapping |
|
| 1099 |
* @param currentTaxonId |
|
| 1100 |
* @param taxonList |
|
| 1101 |
* @param inferredSynonymsDataToBeSaved |
|
| 1102 |
* @return |
|
| 1103 |
*/ |
|
| 1104 |
private Set<TaxonBase> createInferredSynonymsForTaxonList(FaunaEuropaeaImportState state, |
|
| 1105 |
List<TaxonBase> taxonList) {
|
|
| 1106 |
|
|
| 1107 |
Taxon acceptedTaxon; |
|
| 1108 |
Classification classification = null; |
|
| 1109 |
Set<TaxonBase> inferredSynonyms = new HashSet<TaxonBase>(); |
|
| 1110 |
List<Synonym> inferredSynonymsLocal= new ArrayList<Synonym>(); |
|
| 1111 |
boolean localSuccess = true; |
|
| 1112 |
|
|
| 1113 |
HashMap<Integer, TaxonNameBase<?,?>> inferredSynonymsDataToBeSaved = new HashMap<Integer, TaxonNameBase<?,?>>(); |
|
| 1114 |
|
|
| 1115 |
for (TaxonBase<?> taxonBase : taxonList) {
|
|
| 1116 |
|
|
| 1117 |
if (taxonBase.isInstanceOf(Taxon.class)) { // this should always be the case since we should have fetched accepted taxon only, but you never know...
|
|
| 1118 |
acceptedTaxon = CdmBase.deproxy(taxonBase, Taxon.class); |
|
| 1119 |
TaxonNameBase<?,?> taxonName = acceptedTaxon.getName(); |
|
| 1120 |
|
|
| 1121 |
if (taxonName.isInstanceOf(ZoologicalName.class)) {
|
|
| 1122 |
Set<TaxonNode> taxonNodes = acceptedTaxon.getTaxonNodes(); |
|
| 1123 |
TaxonNode singleNode = null; |
|
| 1124 |
|
|
| 1125 |
if (taxonNodes.size() > 0) {
|
|
| 1126 |
// Determine the classification of the current TaxonNode |
|
| 1127 |
|
|
| 1128 |
singleNode = taxonNodes.iterator().next(); |
|
| 1129 |
if (singleNode != null) {
|
|
| 1130 |
classification = singleNode.getClassification(); |
|
| 1131 |
} else {
|
|
| 1132 |
logger.error("A TaxonNode belonging to this accepted Taxon is NULL: " + acceptedTaxon.getUuid() + " (" + acceptedTaxon.getTitleCache() +")");
|
|
| 1133 |
} |
|
| 1134 |
} else {
|
|
| 1135 |
// Classification could not be determined directly from this TaxonNode |
|
| 1136 |
// The stored classification from another TaxonNode is used. It's a simple, but not a failsafe fallback solution. |
|
| 1137 |
if (taxonNodes.size() == 0) {
|
|
| 1138 |
//logger.error("Classification could not be determined directly from this Taxon: " + acceptedTaxon.getUuid() + " is misapplication? "+acceptedTaxon.isMisapplication()+ "). The classification of the last taxon is used");
|
|
| 1139 |
|
|
| 1140 |
} |
|
| 1141 |
} |
|
| 1142 |
|
|
| 1143 |
if (classification != null) {
|
|
| 1144 |
try{
|
|
| 1145 |
TaxonNameBase name = acceptedTaxon.getName(); |
|
| 1146 |
|
|
| 1147 |
//if (name.isSpecies() || name.isInfraSpecific()){
|
|
| 1148 |
inferredSynonymsLocal = getTaxonService().createAllInferredSynonyms(acceptedTaxon, classification, true); |
|
| 1149 |
// logger.info("number of inferred synonyms: " + inferredSynonyms.size());
|
|
| 1150 |
//} |
|
| 1151 |
// inferredSynonyms = getTaxonService().createInferredSynonyms(classification, acceptedTaxon, SynonymRelationshipType.INFERRED_GENUS_OF()); |
|
| 1152 |
if (inferredSynonymsLocal != null) {
|
|
| 1153 |
for (TaxonBase synonym : inferredSynonymsLocal) {
|
|
| 1154 |
// TaxonNameBase<?,?> synonymName = synonym.getName(); |
|
| 1155 |
MarkerType markerType =getUuidMarkerType(PesiTransformer.uuidMarkerGuidIsMissing, state); |
|
| 1156 |
|
|
| 1157 |
synonym.addMarker(Marker.NewInstance(markerType, true)); |
|
| 1158 |
|
|
| 1159 |
|
|
| 1160 |
//get SynonymRelationship and export |
|
| 1161 |
if (((Synonym)synonym).getSynonymRelations().isEmpty() ){
|
|
| 1162 |
SynonymRelationship synRel; |
|
| 1163 |
IdentifiableSource source = ((Synonym)synonym).getSources().iterator().next(); |
|
| 1164 |
if (source.getIdNamespace().contains("Potential combination")){
|
|
| 1165 |
synRel = acceptedTaxon.addSynonym((Synonym)synonym, SynonymRelationshipType.POTENTIAL_COMBINATION_OF()); |
|
| 1166 |
logger.error(synonym.getTitleCache() + " has no synonym relationship to " + acceptedTaxon.getTitleCache() + " type is set to potential combination"); |
|
| 1167 |
} else if (source.getIdNamespace().contains("Inferred Genus")){
|
|
| 1168 |
synRel = acceptedTaxon.addSynonym((Synonym)synonym, SynonymRelationshipType.INFERRED_GENUS_OF()); |
|
| 1169 |
logger.error(synonym.getTitleCache() + " has no synonym relationship to " + acceptedTaxon.getTitleCache() + " type is set to inferred genus"); |
|
| 1170 |
} else if (source.getIdNamespace().contains("Inferred Epithet")){
|
|
| 1171 |
synRel = acceptedTaxon.addSynonym((Synonym)synonym, SynonymRelationshipType.INFERRED_EPITHET_OF()); |
|
| 1172 |
logger.error(synonym.getTitleCache() + " has no synonym relationship to " + acceptedTaxon.getTitleCache() + " type is set to inferred epithet"); |
|
| 1173 |
} else{
|
|
| 1174 |
synRel = acceptedTaxon.addSynonym((Synonym)synonym, SynonymRelationshipType.INFERRED_SYNONYM_OF()); |
|
| 1175 |
logger.error(synonym.getTitleCache() + " has no synonym relationship to " + acceptedTaxon.getTitleCache() + " type is set to inferred synonym"); |
|
| 1176 |
} |
|
| 1177 |
|
|
| 1178 |
|
|
| 1179 |
|
|
| 1180 |
synRel = null; |
|
| 1181 |
} else {
|
|
| 1182 |
for (SynonymRelationship synRel: ((Synonym)synonym).getSynonymRelations()){
|
|
| 1183 |
if (!localSuccess) {
|
|
| 1184 |
logger.error("Synonym relationship export failed " + synonym.getTitleCache() + " accepted taxon: " + acceptedTaxon.getUuid() + " (" + acceptedTaxon.getTitleCache()+")");
|
|
| 1185 |
} else {
|
|
| 1186 |
// logger.info("Synonym relationship successfully exported: " + synonym.getTitleCache() + " " +acceptedTaxon.getUuid() + " (" + acceptedTaxon.getTitleCache()+")");
|
|
| 1187 |
} |
|
| 1188 |
synRel = null; |
|
| 1189 |
} |
|
| 1190 |
} |
|
| 1191 |
|
|
| 1192 |
inferredSynonymsDataToBeSaved.put(synonym.getId(), synonym.getName()); |
|
| 1193 |
} |
|
| 1194 |
|
|
| 1195 |
inferredSynonyms.addAll(inferredSynonymsLocal); |
|
| 1196 |
//logger.info("inferredSet: " + inferredSet.size());
|
|
| 1197 |
//getTaxonService().save(inferredSynonyms); |
|
| 1198 |
//commitTransaction(txStatus); |
|
| 1199 |
|
|
| 1200 |
inferredSynonymsLocal = null; |
|
| 1201 |
|
|
| 1202 |
} |
|
| 1203 |
|
|
| 1204 |
|
|
| 1205 |
}catch(Exception e){
|
|
| 1206 |
logger.error(e.getMessage()); |
|
| 1207 |
e.printStackTrace(); |
|
| 1208 |
} |
|
| 1209 |
} else {
|
|
| 1210 |
logger.error("Classification is NULL. Inferred Synonyms could not be created for this Taxon: " + acceptedTaxon.getUuid() + " (" + acceptedTaxon.getTitleCache() + ")");
|
|
| 1211 |
} |
|
| 1212 |
} else {
|
|
| 1213 |
// logger.error("TaxonName is not a ZoologicalName: " + taxonName.getUuid() + " (" + taxonName.getTitleCache() + ")");
|
|
| 1214 |
} |
|
| 1215 |
} else {
|
|
| 1216 |
logger.error("This TaxonBase is not a Taxon even though it should be: " + taxonBase.getUuid() + " (" + taxonBase.getTitleCache() + ")");
|
|
| 1217 |
} |
|
| 1218 |
} |
|
| 1219 |
//getTaxonService().saveOrUpdate(taxonList); |
|
| 1220 |
taxonList = null; |
|
| 1221 |
return inferredSynonyms; |
|
| 1222 |
} |
|
| 1223 |
|
|
| 990 | 1224 |
|
| 991 | 1225 |
} |
| cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaTransformer.java | ||
|---|---|---|
| 16 | 16 |
|
| 17 | 17 |
import org.apache.log4j.Logger; |
| 18 | 18 |
|
| 19 |
import eu.etaxonomy.cdm.api.application.CdmApplicationUtils; |
|
| 20 | 19 |
import eu.etaxonomy.cdm.api.service.ITermService; |
| 21 | 20 |
import eu.etaxonomy.cdm.io.common.TdwgAreaProvider; |
| 22 | 21 |
import eu.etaxonomy.cdm.model.common.DefinedTerm; |
| 23 |
import eu.etaxonomy.cdm.model.common.LSID; |
|
| 24 | 22 |
import eu.etaxonomy.cdm.model.common.Language; |
| 25 | 23 |
import eu.etaxonomy.cdm.model.common.Representation; |
| 26 | 24 |
import eu.etaxonomy.cdm.model.common.TermType; |
| ... | ... | |
| 399 | 397 |
|
| 400 | 398 |
|
| 401 | 399 |
} |
| 402 |
|
|
| 400 |
|
|
| 403 | 401 |
public final static HashMap<Integer, Language> languageFK2Language = new HashMap<Integer,Language>(); |
| 404 | 402 |
static |
| 405 | 403 |
{
|
| ... | ... | |
| 410 | 408 |
languageFK2Language.put(11, Language.DUTCH_MIDDLE()); |
| 411 | 409 |
languageFK2Language.put(12, Language.ENGLISH()); |
| 412 | 410 |
languageFK2Language.put(16, Language.FRENCH()); |
| 413 |
|
|
| 411 |
|
|
| 414 | 412 |
languageFK2Language.put(18, Language.GERMAN()); |
| 415 | 413 |
languageFK2Language.put(19, Language.GREEK_MODERN()); |
| 416 | 414 |
languageFK2Language.put(23, Language.HEBREW()); |
| ... | ... | |
| 423 | 421 |
languageFK2Language.put(55, Language.SLOVENIAN()); |
| 424 | 422 |
languageFK2Language.put(57, Language.SWEDISH()); |
| 425 | 423 |
languageFK2Language.put(58, Language.TURKISH()); |
| 426 |
|
|
| 424 |
|
|
| 427 | 425 |
languageFK2Language.put(59, Language.UKRAINIAN()); |
| 428 | 426 |
languageFK2Language.put(60, Language.WELSH()); |
| 429 | 427 |
languageFK2Language.put(62, Language.GALICIAN()); |
| 430 | 428 |
//languageFK2Language.put(83, getEnglishUS()); |
| 431 | 429 |
languageFK2Language.put(97, Language.IRISH()); |
| 432 |
|
|
| 433 |
|
|
| 430 |
|
|
| 431 |
|
|
| 434 | 432 |
languageFK2Language.put(100, Language.NORWEGIAN_BOKMOL()); |
| 435 | 433 |
languageFK2Language.put(101, Language.NORWEGIAN_NYNORSK()); |
| 436 |
|
|
| 434 |
|
|
| 437 | 435 |
languageFK2Language.put(102, Language.ARABIC()); |
| 438 | 436 |
languageFK2Language.put(103, Language.ARMENIAN()); |
| 439 |
|
|
| 437 |
|
|
| 440 | 438 |
languageFK2Language.put(104, Language.CATALAN_VALENCIAN()); |
| 441 | 439 |
languageFK2Language.put(105, Language.CHINESE()); |
| 442 | 440 |
languageFK2Language.put(106, Language.ESTONIAN()); |
| 443 | 441 |
languageFK2Language.put(107, Language.FINNISH()); |
| 444 |
|
|
| 442 |
|
|
| 445 | 443 |
languageFK2Language.put(108, Language.GAELIC_SCOTTISH_GAELIC()); |
| 446 | 444 |
languageFK2Language.put(109, Language.JAPANESE()); |
| 447 | 445 |
languageFK2Language.put(110, Language.KOREAN()); |
| ... | ... | |
| 450 | 448 |
languageFK2Language.put(113, Language.PERSIAN()); |
| 451 | 449 |
languageFK2Language.put(114, Language.PORTUGUESE()); |
| 452 | 450 |
languageFK2Language.put(115, Language.ROMANIAN()); |
| 451 |
languageFK2Language.put(116, Language.GAELIC_SCOTTISH_GAELIC()); |
|
| 453 | 452 |
languageFK2Language.put(117, Language.SWAHILI()); |
| 454 | 453 |
languageFK2Language.put(118, Language.SPANISH_CASTILIAN()); |
| 455 |
|
|
| 456 |
|
|
| 457 |
|
|
| 458 | 454 |
|
| 459 | 455 |
|
| 460 |
} |
|
| 456 |
|
|
| 457 |
|
|
| 458 |
|
|
| 459 |
} |
|
| 461 | 460 |
public static UUID getUUIDByAreaAbbr(String abbr){
|
| 462 | 461 |
return abbrToUUID.get(abbr); |
| 463 | 462 |
} |
| ... | ... | |
| 498 | 497 |
} |
| 499 | 498 |
return nomStatusTempNamed; |
| 500 | 499 |
} |
| 501 |
|
|
| 500 |
|
|
| 502 | 501 |
public static Language getEnglishUS(ITermService termService){
|
| 503 | 502 |
if (langEnglishUS == null){
|
| 504 | 503 |
langEnglishUS = (Language)termService.find(uuidEnglishUS); |
| ... | ... | |
| 520 | 519 |
if (taxonomicSpecialistType == null){
|
| 521 | 520 |
taxonomicSpecialistType = (DefinedTerm)termService.find(uuidTaxonomicSpecialistType); |
| 522 | 521 |
if (taxonomicSpecialistType == null){
|
| 523 |
logger.info("create associated specialist type");
|
|
| 522 |
logger.info("create taxonomic specialist type");
|
|
| 524 | 523 |
taxonomicSpecialistType = DefinedTerm.NewInstance(TermType.TaxonNodeAgentRelationType, "taxonomic specialist", "taxonomic specialist", "TS"); |
| 525 | 524 |
|
| 526 | 525 |
taxonomicSpecialistType.setUuid(uuidTaxonomicSpecialistType); |
| ... | ... | |
| 546 | 545 |
} |
| 547 | 546 |
return groupCoordinatorType; |
| 548 | 547 |
} |
| 549 |
|
|
| 550 |
|
|
| 551 |
|
|
| 548 |
|
|
| 549 |
|
|
| 550 |
|
|
| 552 | 551 |
/** |
| 553 | 552 |
* @return |
| 554 | 553 |
*/ |
| ... | ... | |
| 567 | 566 |
} |
| 568 | 567 |
public static Language langFK2Language( |
| 569 | 568 |
Integer languageFk) {
|
| 570 |
|
|
| 569 |
|
|
| 571 | 570 |
Language result = languageFK2Language.get(languageFk); |
| 572 |
|
|
| 571 |
|
|
| 573 | 572 |
return result; |
| 574 | 573 |
} |
| 575 | 574 |
|
| cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/faunaEuropaea/FaunaEuropaeaVernacularName.java | ||
|---|---|---|
| 3 | 3 |
import java.util.UUID; |
| 4 | 4 |
|
| 5 | 5 |
public class FaunaEuropaeaVernacularName {
|
| 6 |
|
|
| 6 |
|
|
| 7 | 7 |
String vernacularName; |
| 8 | 8 |
int languageFk; |
| 9 | 9 |
String languageCache; |
| ... | ... | |
| 11 | 11 |
String guid; |
| 12 | 12 |
String source; |
| 13 | 13 |
String area = ""; |
| 14 |
|
|
| 15 |
|
|
| 14 |
|
|
| 15 |
|
|
| 16 | 16 |
public FaunaEuropaeaVernacularName(String vernacularName, int languageFK, String languageCache, UUID taxonUuid, String guid, String source){
|
| 17 | 17 |
this.guid=guid; |
| 18 | 18 |
this.languageCache = languageCache; |
| 19 |
this.languageFk = languageFk;
|
|
| 19 |
this.languageFk = languageFK;
|
|
| 20 | 20 |
this.taxonUuid = taxonUuid; |
| 21 | 21 |
this.vernacularName = vernacularName; |
| 22 | 22 |
this.source = source; |
| 23 |
|
|
| 23 |
|
|
| 24 | 24 |
} |
| 25 | 25 |
|
| 26 | 26 |
public FaunaEuropaeaVernacularName() {
|
| 27 |
|
|
| 27 |
|
|
| 28 | 28 |
} |
| 29 |
|
|
| 30 |
|
|
| 29 |
|
|
| 30 |
|
|
| 31 | 31 |
public String getArea() {
|
| 32 | 32 |
return area; |
| 33 | 33 |
} |
| ... | ... | |
| 83 | 83 |
public void setSource(String source) {
|
| 84 | 84 |
this.source = source; |
| 85 | 85 |
} |
| 86 |
|
|
| 87 |
|
|
| 86 |
|
|
| 87 |
|
|
| 88 | 88 |
} |
Also available in: Unified diff
minor Fauna Eu changes