From: Andreas Müller Date: Wed, 21 Sep 2011 14:29:34 +0000 (+0000) Subject: Updates to BM import to integrate AlgaTerra X-Git-Tag: cdmlib-parent-3.0.9~286 X-Git-Url: https://dev.e-taxonomy.eu/gitweb/cdmlib.git/commitdiff_plain/15cb83a7cd6c4bc75d3ac1d958718005bd9f993a Updates to BM import to integrate AlgaTerra --- diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelFactsImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelFactsImport.java index fb2aec6618..2eac7b90db 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelFactsImport.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelFactsImport.java @@ -199,7 +199,6 @@ public class BerlinModelFactsImport extends BerlinModelImportBase { String fact = CdmUtils.Nz(rs.getString("Fact")); String notes = CdmUtils.Nz(rs.getString("notes")); Boolean doubtfulFlag = rs.getBoolean("DoubtfulFlag"); - Boolean publishFlag = rs.getBoolean("publishFlag"); TaxonBase taxonBase = getTaxon(taxonMap, taxonIdObj, taxonId); Feature feature = getFeature(featureMap, categoryFkObj, categoryFk) ; @@ -314,7 +313,13 @@ public class BerlinModelFactsImport extends BerlinModelImportBase { textData.addMarker(Marker.NewInstance(MarkerType.IS_DOUBTFUL(), true)); } //publisheFlag - textData.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), publishFlag)); + String strPublishFlag = "publishFlag"; + boolean publishFlagExists = state.getConfig().getSource().checkColumnExists(dbTableName, strPublishFlag); + if (publishFlagExists){ + Boolean publishFlag = rs.getBoolean(strPublishFlag); + textData.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), publishFlag)); + } + //Sequence Integer sequence = rs.getInt("Sequence"); if (sequence != null && sequence != 999){ diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonImport.java index 2240556591..788618e221 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonImport.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonImport.java @@ -223,10 +223,14 @@ public class BerlinModelTaxonImport extends BerlinModelImportBase { taxonBase.setUseNameCache(true); } //publisheFlag - Boolean publishFlag = rs.getBoolean("PublishFlag"); - if (config.getTaxonPublishMarker().doMark(publishFlag)){ - taxonBase.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), publishFlag)); + boolean publishFlagExists = state.getConfig().getSource().checkColumnExists("PTaxon", "PublishFlag"); + if (publishFlagExists){ + Boolean publishFlag = rs.getBoolean("PublishFlag"); + if (config.getTaxonPublishMarker().doMark(publishFlag)){ + taxonBase.addMarker(Marker.NewInstance(MarkerType.PUBLISH(), publishFlag)); + } } + //Notes doIdCreatedUpdatedNotes(state, taxonBase, rs, taxonId, NAMESPACE); diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonNameImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonNameImport.java index fcf82ddb87..f2bfc4c2ae 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonNameImport.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonNameImport.java @@ -17,6 +17,8 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import javax.mail.MethodNotSupportedException; + import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -187,10 +189,18 @@ public class BerlinModelTaxonNameImport extends BerlinModelImportBase { success &= makeNomenclaturalReference(config, taxonNameBase, nameId, rs, partitioner); //Source_Acc - String sourceAcc = rs.getString("Source_Acc"); - if (CdmUtils.isNotEmpty(sourceAcc)){ - ExtensionType sourceAccExtensionType = getExtensionType(state, SOURCE_ACC_UUID, "Source_Acc","Source_Acc","Source_Acc"); - Extension datesExtension = Extension.NewInstance(taxonNameBase, sourceAcc, sourceAccExtensionType); + boolean colExists = true; + try { + colExists = state.getConfig().getSource().checkColumnExists("Name", "Source_Acc"); + } catch (MethodNotSupportedException e) { + logger.debug("Source does not support 'checkColumnExists'"); + } + if (colExists){ + String sourceAcc = rs.getString("Source_Acc"); + if (CdmUtils.isNotEmpty(sourceAcc)){ + ExtensionType sourceAccExtensionType = getExtensionType(state, SOURCE_ACC_UUID, "Source_Acc","Source_Acc","Source_Acc"); + Extension datesExtension = Extension.NewInstance(taxonNameBase, sourceAcc, sourceAccExtensionType); + } } //created, notes diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonRelationImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonRelationImport.java index 07b7fb3bd8..6575eb16bc 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonRelationImport.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonRelationImport.java @@ -104,31 +104,36 @@ public class BerlinModelTaxonRelationImport extends BerlinModelImportBase { try { while (rs.next()){ - if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));} - - Object ptRefFkObj = rs.getObject("PTRefFk"); - String ptRefFk= String.valueOf(ptRefFkObj); - Reference ref = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, ptRefFk); - - rs.getString("RefCache"); - String treeName = "Classification - No Name"; - String refCache = rs.getString("RefCache"); - if (CdmUtils.isNotEmpty(refCache)){ - treeName = refCache; - } - if (ref != null && CdmUtils.isNotEmpty(ref.getTitleCache())){ - treeName = ref.getTitleCache(); - } - Classification tree = Classification.NewInstance(treeName); - tree.setReference(ref); - if (i == 1 && state.getConfig().getClassificationUuid() != null){ - tree.setUuid(state.getConfig().getClassificationUuid()); + try { + if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));} + + Object ptRefFkObj = rs.getObject("PTRefFk"); + String ptRefFk= String.valueOf(ptRefFkObj); + Reference ref = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, ptRefFk); + + rs.getString("RefCache"); + String treeName = "Classification - No Name"; + String refCache = rs.getString("RefCache"); + if (CdmUtils.isNotEmpty(refCache)){ + treeName = refCache; + } + if (ref != null && CdmUtils.isNotEmpty(ref.getTitleCache())){ + treeName = ref.getTitleCache(); + } + Classification tree = Classification.NewInstance(treeName); + tree.setReference(ref); + if (i == 1 && state.getConfig().getClassificationUuid() != null){ + tree.setUuid(state.getConfig().getClassificationUuid()); + } + IdentifiableSource identifiableSource = IdentifiableSource.NewInstance(ptRefFk, TREE_NAMESPACE); + tree.addSource(identifiableSource); + + getClassificationService().save(tree); + state.putClassificationUuidInt((Integer)ptRefFkObj, tree); + } catch (Exception e) { + logger.error("Error in BerlinModleTaxonRelationImport.makeClassifications: " + e.getMessage()); + e.printStackTrace(); } - IdentifiableSource identifiableSource = IdentifiableSource.NewInstance(ptRefFk, TREE_NAMESPACE); - tree.addSource(identifiableSource); - - getClassificationService().save(tree); - state.putClassificationUuidInt((Integer)ptRefFkObj, tree); } } catch (SQLException e) { logger.error("Error in BerlinModleTaxonRelationImport.makeClassifications: " + e.getMessage()); @@ -202,101 +207,106 @@ public class BerlinModelTaxonRelationImport extends BerlinModelImportBase { while (rs.next()){ if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));} - + int relPTaxonId = rs.getInt("RelPTaxonId"); - int taxon1Id = rs.getInt("taxon1Id"); - int taxon2Id = rs.getInt("taxon2Id"); - Object relRefFkObj = rs.getObject("relRefFk"); - int treeRefFk = rs.getInt("treeRefFk"); - int relQualifierFk = rs.getInt("relQualifierFk"); - String notes = rs.getString("notes"); - boolean isConceptRelationship = rs.getBoolean("is_concept_relation"); - - TaxonBase taxon1 = taxonMap.get(String.valueOf(taxon1Id)); - TaxonBase taxon2 = taxonMap.get(String.valueOf(taxon2Id)); - - String refFk = String.valueOf(relRefFkObj); - Reference citation = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, refFk); - - String microcitation = null; //does not exist in RelPTaxon - - if (taxon2 != null && taxon1 != null){ - if (!(taxon2 instanceof Taxon)){ - logger.error("TaxonBase (ID = " + taxon2.getId()+ ", RIdentifier = " + taxon2Id + ") can't be casted to Taxon"); - success = false; - continue; - } - AnnotatableEntity taxonRelationship = null; - Taxon toTaxon = (Taxon)taxon2; - if (isTaxonRelationship(relQualifierFk)){ - if (!(taxon1 instanceof Taxon)){ - logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Taxon"); - success = false; - continue; - } - Taxon fromTaxon = (Taxon)taxon1; - if (relQualifierFk == TAX_REL_IS_INCLUDED_IN){ - taxonRelationship = makeTaxonomicallyIncluded(state, classificationMap, treeRefFk, fromTaxon, toTaxon, citation, microcitation); - }else if (relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){ - taxonRelationship = toTaxon.addMisappliedName(fromTaxon, citation, microcitation); - } - }else if (isSynonymRelationship(relQualifierFk)){ - if (!(taxon1 instanceof Synonym)){ - logger.error("Taxon (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Synonym"); + try { + int taxon1Id = rs.getInt("taxon1Id"); + int taxon2Id = rs.getInt("taxon2Id"); + Object relRefFkObj = rs.getObject("relRefFk"); + int treeRefFk = rs.getInt("treeRefFk"); + int relQualifierFk = rs.getInt("relQualifierFk"); + String notes = rs.getString("notes"); + boolean isConceptRelationship = rs.getBoolean("is_concept_relation"); + + TaxonBase taxon1 = taxonMap.get(String.valueOf(taxon1Id)); + TaxonBase taxon2 = taxonMap.get(String.valueOf(taxon2Id)); + + String refFk = String.valueOf(relRefFkObj); + Reference citation = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, refFk); + + String microcitation = null; //does not exist in RelPTaxon + + if (taxon2 != null && taxon1 != null){ + if (!(taxon2 instanceof Taxon)){ + logger.error("TaxonBase (ID = " + taxon2.getId()+ ", RIdentifier = " + taxon2Id + ") can't be casted to Taxon"); success = false; continue; } - Synonym synonym = (Synonym)taxon1; - SynonymRelationship synRel = getSynRel(relQualifierFk, toTaxon, synonym, citation, microcitation); - taxonRelationship = synRel; - - if (relQualifierFk == TAX_REL_IS_SYNONYM_OF || - relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF || - relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF){ - addProParteAndPartial(synRel, synonym, config); - }else if (relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF || - relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF || - relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ){ - synRel.setProParte(true); - }else if(relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF || - relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF || - relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF ){ - synRel.setPartial(true); - }else{ - success = false; - logger.warn("Proparte/Partial not yet implemented for TaxonRelationShipType " + relQualifierFk); - } - }else if (isConceptRelationship){ - ResultWrapper isInverse = new ResultWrapper(); - try { - TaxonRelationshipType relType = BerlinModelTransformer.taxonRelId2TaxonRelType(relQualifierFk, isInverse); - if (! (taxon1 instanceof Taxon)){ - success = false; + AnnotatableEntity taxonRelationship = null; + Taxon toTaxon = (Taxon)taxon2; + if (isTaxonRelationship(relQualifierFk)){ + if (!(taxon1 instanceof Taxon)){ logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Taxon"); + success = false; + continue; + } + Taxon fromTaxon = (Taxon)taxon1; + if (relQualifierFk == TAX_REL_IS_INCLUDED_IN){ + taxonRelationship = makeTaxonomicallyIncluded(state, classificationMap, treeRefFk, fromTaxon, toTaxon, citation, microcitation); + }else if (relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){ + taxonRelationship = toTaxon.addMisappliedName(fromTaxon, citation, microcitation); + } + }else if (isSynonymRelationship(relQualifierFk)){ + if (!(taxon1 instanceof Synonym)){ + logger.warn("Validated: Taxon (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Synonym"); + success = false; + continue; + } + Synonym synonym = (Synonym)taxon1; + SynonymRelationship synRel = getSynRel(relQualifierFk, toTaxon, synonym, citation, microcitation); + taxonRelationship = synRel; + + if (relQualifierFk == TAX_REL_IS_SYNONYM_OF || + relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF || + relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF){ + addProParteAndPartial(synRel, synonym, config); + }else if (relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF || + relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF || + relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ){ + synRel.setProParte(true); + }else if(relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF || + relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF || + relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF ){ + synRel.setPartial(true); }else{ - Taxon fromTaxon = (Taxon)taxon1; - taxonRelationship = fromTaxon.addTaxonRelation(toTaxon, relType, citation, microcitation); + success = false; + logger.warn("Proparte/Partial not yet implemented for TaxonRelationShipType " + relQualifierFk); + } + }else if (isConceptRelationship){ + ResultWrapper isInverse = new ResultWrapper(); + try { + TaxonRelationshipType relType = BerlinModelTransformer.taxonRelId2TaxonRelType(relQualifierFk, isInverse); + if (! (taxon1 instanceof Taxon)){ + success = false; + logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Taxon"); + }else{ + Taxon fromTaxon = (Taxon)taxon1; + taxonRelationship = fromTaxon.addTaxonRelation(toTaxon, relType, citation, microcitation); + } + } catch (UnknownCdmTypeException e) { + //TODO other relationships + logger.warn("TaxonRelationShipType " + relQualifierFk + " (conceptRelationship) not yet implemented"); + success = false; } - } catch (UnknownCdmTypeException e) { - //TODO other relationships - logger.warn("TaxonRelationShipType " + relQualifierFk + " (conceptRelationship) not yet implemented"); + }else { + //TODO + logger.warn("TaxonRelationShipType " + relQualifierFk + " not yet implemented"); success = false; } - }else { + + doNotes(taxonRelationship, notes); + taxaToSave.add(taxon2); + + //TODO + //etc. + }else{ //TODO - logger.warn("TaxonRelationShipType " + relQualifierFk + " not yet implemented"); + logger.warn("Taxa for RelPTaxon " + relPTaxonId + " do not exist in store"); success = false; } - - doNotes(taxonRelationship, notes); - taxaToSave.add(taxon2); - - //TODO - //etc. - }else{ - //TODO - logger.warn("Taxa for RelPTaxon " + relPTaxonId + " do not exist in store"); - success = false; + } catch (Exception e) { + logger.error("Exception occurred when trying to handle taxon relationship " + relPTaxonId + ":" + e.getMessage()); +// e.printStackTrace(); } } }catch(SQLException e){ diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelFactsImportValidator.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelFactsImportValidator.java index f73a90c0dc..c9558c8f77 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelFactsImportValidator.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelFactsImportValidator.java @@ -33,17 +33,16 @@ public class BerlinModelFactsImportValidator implements IOValidator 0){ + System.out.println("========================================================"); + logger.warn("There are "+count+" Facts with attached synonyms."); + + System.out.println("========================================================"); + } + return result; + } catch (SQLException e) { + e.printStackTrace(); + return false; + } + + } } diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelRefDetailImportValidator.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelRefDetailImportValidator.java index 7156b7c9b8..239fd2468c 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelRefDetailImportValidator.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelRefDetailImportValidator.java @@ -13,6 +13,8 @@ package eu.etaxonomy.cdm.io.berlinModel.in.validation; import java.sql.ResultSet; import java.sql.SQLException; +import javax.mail.MethodNotSupportedException; + import org.apache.log4j.Logger; import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState; @@ -74,23 +76,27 @@ public class BerlinModelRefDetailImportValidator implements IOValidator '')"; - ResultSet rs = source.getResultSet(strQuery); - rs.next(); - int n; - n = rs.getInt("n"); - if (n > 0){ - System.out.println("========================================================"); - logger.warn("There are " + n + " RefDetails with an idInSource. IdInSources are not supported yet"); - System.out.println("========================================================"); - success = false; + if (source.checkColumnExists("RefDetail", "idInSource")){ + String strQuery = + "SELECT count(*) AS n FROM RefDetail " + + " WHERE (IdInSource IS NOT NULL) AND (RTRIM(LTRIM(IdInSource)) <> '')"; + ResultSet rs = source.getResultSet(strQuery); + rs.next(); + int n; + n = rs.getInt("n"); + if (n > 0){ + System.out.println("========================================================"); + logger.warn("There are " + n + " RefDetails with an idInSource. IdInSources are not supported yet"); + System.out.println("========================================================"); + success = false; + } } + } catch (SQLException e) { e.printStackTrace(); + } catch (MethodNotSupportedException e) { + logger.debug("Source does not support checking existance of 'idInSource' column"); } return success; } diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelTaxonImportValidator.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelTaxonImportValidator.java index 8e30bb2404..276377002f 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelTaxonImportValidator.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelTaxonImportValidator.java @@ -31,8 +31,7 @@ public class BerlinModelTaxonImportValidator implements IOValidator