Revision 21e704da
Added by Andreas Müller about 5 years ago
| app-import/src/main/java/eu/etaxonomy/cdm/app/common/CdmDestinations.java | ||
|---|---|---|
| 308 | 308 |
return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null); |
| 309 | 309 |
} |
| 310 | 310 |
|
| 311 |
public static ICdmDataSource cdm_phycobank_production(){
|
|
| 312 |
DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL; |
|
| 313 |
String cdmServer = "160.45.63.171"; |
|
| 314 |
String cdmDB = "cdm_production_phycobank"; |
|
| 315 |
String cdmUserName = "edit"; |
|
| 316 |
return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null); |
|
| 317 |
} |
|
| 318 |
|
|
| 311 | 319 |
public static ICdmDataSource cdm_production_flora_cuba(){
|
| 312 | 320 |
DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL; |
| 313 | 321 |
String cdmServer = "160.45.63.171"; |
| app-import/src/main/java/eu/etaxonomy/cdm/app/phycobank/PhycobankHigherClassificationActivator.java | ||
|---|---|---|
| 22 | 22 |
import eu.etaxonomy.cdm.io.common.CdmDefaultImport; |
| 23 | 23 |
import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK; |
| 24 | 24 |
import eu.etaxonomy.cdm.io.phycobank.PhycobankHigherClassificationImportConfigurator; |
| 25 |
import eu.etaxonomy.cdm.model.agent.Person; |
|
| 26 | 25 |
import eu.etaxonomy.cdm.model.common.VerbatimTimePeriod; |
| 27 | 26 |
import eu.etaxonomy.cdm.model.reference.Reference; |
| 28 | 27 |
import eu.etaxonomy.cdm.model.reference.ReferenceFactory; |
| ... | ... | |
| 42 | 41 |
// ==================================================================================== |
| 43 | 42 |
|
| 44 | 43 |
//database validation status (create, update, validate ...) |
| 45 |
static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE; //
|
|
| 44 |
static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE; //
|
|
| 46 | 45 |
|
| 47 |
static ICdmDataSource cdmDestination = CdmDestinations.localH2(); |
|
| 46 |
// static ICdmDataSource cdmDestination = CdmDestinations.localH2();
|
|
| 48 | 47 |
// static ICdmDataSource cdmDestination = CdmDestinations.cdm_local_test_mysql(); |
| 49 | 48 |
// static ICdmDataSource cdmDestination = CdmDestinations.cdm_test_phycobank(); |
| 50 |
//TODO: Die 3 Syllabus noch richtig als 3 Referenzen anlegen (muss Henning machen), hier dann die jeweilige UUID |
|
| 51 |
// verwenden und unten den worksheetNamen entsprechend anpassen |
|
| 52 |
static Reference secRef = getSecReference_Frey(); |
|
| 53 |
// static Reference secRef = getSecReference_WoRMS(); |
|
| 54 |
// static String worksheetName = secRef.equals(getSecReference_WoRMS())? "WoRMS" : |
|
| 55 |
// secRef.equals(getSecReference_Frey())? "HigherRanksEntwurfNeu" : null; |
|
| 49 |
static ICdmDataSource cdmDestination = CdmDestinations.cdm_phycobank_production(); |
|
| 50 |
|
|
| 56 | 51 |
static String worksheetName = "Syllabus2_1"; |
| 57 | 52 |
|
| 58 | 53 |
static UUID uuidRefPhycobank = UUID.fromString("8058a5ec-60ee-4a04-8c17-5623e3a4795c");
|
| ... | ... | |
| 60 | 55 |
//check - import |
| 61 | 56 |
static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK; |
| 62 | 57 |
|
| 58 |
|
|
| 63 | 59 |
private void doImport(ICdmDataSource cdmDestination){
|
| 60 |
hbm2dll = (cdmDestination == CdmDestinations.cdm_phycobank_production()|| cdmDestination == CdmDestinations.cdm_test_phycobank())? |
|
| 61 |
DbSchemaValidation.VALIDATE : hbm2dll; |
|
| 64 | 62 |
|
| 65 | 63 |
URI source = fileURI(); |
| 66 | 64 |
Reference sourceRef = getSourceReference(); |
| ... | ... | |
| 71 | 69 |
config.setCheck(check); |
| 72 | 70 |
config.setDbSchemaValidation(hbm2dll); |
| 73 | 71 |
config.setSourceReference(sourceRef); |
| 74 |
config.setSecReference(secRef); |
|
| 75 | 72 |
config.setPhycobankReference(getPhycobankReference()); |
| 76 | 73 |
config.setProgressMonitor(DefaultProgressMonitor.NewInstance()); |
| 77 | 74 |
|
| ... | ... | |
| 79 | 76 |
myImport.invoke(config); |
| 80 | 77 |
|
| 81 | 78 |
System.exit(0); |
| 82 |
|
|
| 83 |
} |
|
| 84 |
|
|
| 85 |
private static Reference getSecReference_Frey() {
|
|
| 86 |
Reference result = ReferenceFactory.newBook(); |
|
| 87 |
result.setTitle("Syllabus of the plant families");
|
|
| 88 |
result.setDatePublished(VerbatimTimePeriod.NewVerbatimInstance(2015)); |
|
| 89 |
result.setPublisher("Borntraeger");
|
|
| 90 |
result.setPlacePublished("Stuttgart");
|
|
| 91 |
Person author = Person.NewInstance(); |
|
| 92 |
author.setFamilyName("Frey");
|
|
| 93 |
author.setInitials("W.");
|
|
| 94 |
result.setAuthorship(author); |
|
| 95 |
result.setUuid(UUID.fromString("2b4a3a67-e432-4d6b-b716-081045179df9"));
|
|
| 96 |
return result; |
|
| 97 |
} |
|
| 98 |
|
|
| 99 |
private static Reference getSecReference_WoRMS() {
|
|
| 100 |
Reference result = ReferenceFactory.newDatabase(); |
|
| 101 |
result.setTitle("WoRMS World Register of Marine Species");
|
|
| 102 |
result.setDatePublished(TimePeriodParser.parseStringVerbatim("2018-04-20"));
|
|
| 103 |
result.setUri(URI.create("http://www.marinespecies.org/index.php"));
|
|
| 104 |
result.setUuid(UUID.fromString("b33daeb0-8770-4ee2-92d0-80aaa87bfba2"));
|
|
| 105 |
return result; |
|
| 106 | 79 |
} |
| 107 | 80 |
|
| 108 | 81 |
private static Reference getPhycobankReference() {
|
| ... | ... | |
| 123 | 96 |
} |
| 124 | 97 |
|
| 125 | 98 |
public static String fileName(){
|
| 126 |
// return "Algen_Syllabus_NormalImplied_Test.xlsx"; |
|
| 127 |
return "Algen_Syllabus_Produktion_corr.xlsx"; |
|
| 99 |
return "Algen_Syllabus_Produktion_2019-01-08_Import_2.xlsx"; |
|
| 128 | 100 |
} |
| 129 | 101 |
public static String filePath(){
|
| 130 | 102 |
return "file:////BGBM-PESIHPC/Phycobank/"; |
| app-import/src/main/java/eu/etaxonomy/cdm/io/phycobank/PhycobankHigherClassificationExcelImport.java | ||
|---|---|---|
| 24 | 24 |
import eu.etaxonomy.cdm.model.name.TaxonName; |
| 25 | 25 |
import eu.etaxonomy.cdm.model.name.TaxonNameFactory; |
| 26 | 26 |
import eu.etaxonomy.cdm.model.reference.Reference; |
| 27 |
import eu.etaxonomy.cdm.model.reference.ReferenceFactory; |
|
| 27 | 28 |
import eu.etaxonomy.cdm.model.taxon.Classification; |
| 28 | 29 |
import eu.etaxonomy.cdm.model.taxon.Taxon; |
| 29 | 30 |
import eu.etaxonomy.cdm.model.taxon.TaxonNode; |
| ... | ... | |
| 44 | 45 |
private static final Logger logger = Logger.getLogger(PhycobankHigherClassificationExcelImport.class); |
| 45 | 46 |
|
| 46 | 47 |
private static final String GENUS = "genus"; |
| 48 |
private static final String SUBFAMILIA = "subfam"; |
|
| 47 | 49 |
private static final String FAMILIA = "familia"; |
| 48 | 50 |
private static final String SUBORDO = "subordo"; |
| 49 | 51 |
private static final String ORDO = "ordo"; |
| ... | ... | |
| 113 | 115 |
List<RankedUninomial> rankedUninomials, String line) {
|
| 114 | 116 |
|
| 115 | 117 |
if (rankedUninomials.isEmpty()){
|
| 116 |
return getClassification(state).getRootNode(); |
|
| 118 |
return getClassification(state, line).getRootNode();
|
|
| 117 | 119 |
}else{
|
| 118 | 120 |
RankedUninomial rankedUninomial = rankedUninomials.get(0); |
| 119 | 121 |
rankedUninomials.remove(0); |
| 120 | 122 |
Taxon taxon = getOrMakeTaxon(state, rankedUninomial, line); |
| 121 |
TaxonNode existingNode = taxon.getTaxonNode(getClassification(state)); |
|
| 123 |
TaxonNode existingNode = taxon.getTaxonNode(getClassification(state, line));
|
|
| 122 | 124 |
TaxonNode existingHigherNode = existingNode == null? null : existingNode.getParent(); |
| 123 | 125 |
//recursive call |
| 124 | 126 |
TaxonNode createdHigher = createOrVerifyRankedUninomials(state, rankedUninomials, line); |
| ... | ... | |
| 138 | 140 |
logger.warn(line + ": Created higher node is null. This should not happen. Please check classification and concept relationships."); |
| 139 | 141 |
return null; |
| 140 | 142 |
}else{
|
| 141 |
existingNode = createdHigher.addChildTaxon(taxon, getSecReference(state), line); |
|
| 143 |
existingNode = createdHigher.addChildTaxon(taxon, getSecReference(state, line), line);
|
|
| 142 | 144 |
getTaxonNodeService().saveOrUpdate(existingNode); |
| 143 | 145 |
} |
| 144 | 146 |
} |
| 145 |
makeConceptRelation(state, existingNode, line); |
|
| 147 |
makeConceptRelation(state, existingNode, line, line);
|
|
| 146 | 148 |
return existingNode; |
| 147 | 149 |
} |
| 148 | 150 |
} |
| ... | ... | |
| 187 | 189 |
|
| 188 | 190 |
List<RankedUninomial> result = new ArrayList<>(); |
| 189 | 191 |
addRankedUninomial(result, record, GENUS, Rank.GENUS()); |
| 192 |
addRankedUninomial(result, record, SUBFAMILIA, Rank.SUBFAMILY()); |
|
| 190 | 193 |
addRankedUninomial(result, record, FAMILIA, Rank.FAMILY()); |
| 191 | 194 |
addRankedUninomial(result, record, SUBORDO, Rank.SUBORDER()); |
| 192 | 195 |
addRankedUninomial(result, record, ORDO, Rank.ORDER()); |
| ... | ... | |
| 240 | 243 |
taxa.addAll(getReferencedTaxa(name, state.getConfig().getPhycobankReference())); |
| 241 | 244 |
} |
| 242 | 245 |
if (taxa.isEmpty()){
|
| 243 |
logger.warn(line + ": More than 1 name matches, but no matching taxon exists. Create new taxon with arbitrary name."); |
|
| 246 |
logger.warn(line + ": (" +rankedUninomial.uninomial + ")More than 1 name matches, but no matching taxon exists. Create new taxon with arbitrary name.");
|
|
| 244 | 247 |
TaxonName name = names.get(0); |
| 245 | 248 |
taxon = getOrCreateTaxon(state, name, phycobankRef, line); |
| 246 | 249 |
}else if (taxa.size() == 1){
|
| 247 | 250 |
taxon = taxa.get(0); |
| 248 | 251 |
}else{
|
| 249 |
logger.warn(line + ": More than 1 taxon matches, take arbitrary one. This is unexpected and could be improved in code by " |
|
| 252 |
logger.warn(line + ": (" +rankedUninomial.uninomial + ") More than 1 taxon matches, take arbitrary one. This is unexpected and could be improved in code by "
|
|
| 250 | 253 |
+ "also checking parent relationships."); |
| 251 | 254 |
taxon = taxa.get(0); |
| 252 | 255 |
} |
| ... | ... | |
| 262 | 265 |
* @param state |
| 263 | 266 |
* @return |
| 264 | 267 |
*/ |
| 265 |
private Classification getClassification(SimpleExcelTaxonImportState<CONFIG> state) {
|
|
| 268 |
private Classification getClassification(SimpleExcelTaxonImportState<CONFIG> state, String line) {
|
|
| 266 | 269 |
Classification result = null; |
| 267 | 270 |
List<Classification> classifications = getClassificationService().list(null, null, null, null, null); |
| 268 |
Reference sec = getSecReference(state); |
|
| 271 |
Reference sec = getSecReference(state, line);
|
|
| 269 | 272 |
for (Classification classification: classifications){
|
| 270 | 273 |
if (classification.getCitation() != null && classification.getCitation().equals(sec)){
|
| 271 | 274 |
result = classification; |
| ... | ... | |
| 283 | 286 |
/** |
| 284 | 287 |
* @param state |
| 285 | 288 |
*/ |
| 286 |
private Reference getSecReference(SimpleExcelTaxonImportState<CONFIG> state) {
|
|
| 287 |
UUID uuid = state.getConfig().getSecReference().getUuid(); |
|
| 289 |
private Reference getSecReference(SimpleExcelTaxonImportState<CONFIG> state, String line) {
|
|
| 290 |
String uuidStr = state.getOriginalRecord().get("reference");
|
|
| 291 |
UUID uuid = uuidStr == null? null:UUID.fromString(uuidStr); |
|
| 292 |
if (uuid == null){
|
|
| 293 |
logger.warn(line + ": reference uuid missing"); |
|
| 294 |
uuid = state.getConfig().getSecReference().getUuid(); |
|
| 295 |
} |
|
| 288 | 296 |
if (secReference == null || !secReference.getUuid().equals(uuid)){
|
| 289 | 297 |
secReference = getReferenceService().find(uuid); |
| 290 | 298 |
if (secReference == null){
|
| 291 |
secReference = state.getConfig().getSecReference(); |
|
| 299 |
secReference = ReferenceFactory.newGeneric(); |
|
| 300 |
logger.warn(line + ": reference could not be found in database"); |
|
| 301 |
secReference.setUuid(uuid); |
|
| 292 | 302 |
getReferenceService().save(secReference); |
| 293 |
|
|
| 294 | 303 |
} |
| 295 | 304 |
} |
| 296 | 305 |
return secReference; |
| ... | ... | |
| 404 | 413 |
* @param microCitation |
| 405 | 414 |
*/ |
| 406 | 415 |
private void makeConceptRelation(SimpleExcelTaxonImportState<CONFIG> state, |
| 407 |
TaxonNode childNode, String microCitation) {
|
|
| 416 |
TaxonNode childNode, String microCitation, String line) {
|
|
| 408 | 417 |
|
| 409 | 418 |
Taxon child = childNode.getTaxon(); |
| 410 | 419 |
Taxon parent = childNode.getParent().getTaxon(); |
| ... | ... | |
| 412 | 421 |
return; |
| 413 | 422 |
} |
| 414 | 423 |
|
| 415 |
Reference sec = getSecReference(state); |
|
| 424 |
Reference sec = getSecReference(state, line);
|
|
| 416 | 425 |
Set<TaxonRelationship> rels = child.getRelationsFromThisTaxon(); |
| 417 | 426 |
boolean hasRelation = false; |
| 418 | 427 |
for (TaxonRelationship rel : rels){
|
| app-import/src/main/java/eu/etaxonomy/cdm/io/phycobank/PhycobankHigherClassificationImportConfigurator.java | ||
|---|---|---|
| 48 | 48 |
return new PhycobankHigherClassificationImportState(this); |
| 49 | 49 |
} |
| 50 | 50 |
|
| 51 |
@SuppressWarnings("unchecked")
|
|
| 51 | 52 |
@Override |
| 52 | 53 |
protected void makeIoClassList() {
|
| 53 | 54 |
ioClassList = new Class[]{
|
Also available in: Unified diff
ref #7420 last changes to Phycobank higher classification import