X-Git-Url: https://dev.e-taxonomy.eu/gitweb/cdmlib.git/blobdiff_plain/416cc6ca3865ca025bd44a865e4a10d79e3ea5e3..fad74844bd8face7f35484034c522691fb4c17b5:/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java diff --git a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java index 421b2eda9f..589a205e7e 100644 --- a/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java +++ b/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java @@ -20,9 +20,6 @@ import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_PART_OF import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_PRINT_SERIES; import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_UNKNOWN; import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_WEBSITE; -import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.ALL; -import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.CONCEPT_REFERENCES; -import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.NOMENCLATURAL; import static eu.etaxonomy.cdm.io.common.ImportHelper.NO_OVERWRITE; import static eu.etaxonomy.cdm.io.common.ImportHelper.OBLIGATORY; import static eu.etaxonomy.cdm.io.common.ImportHelper.OVERWRITE; @@ -58,6 +55,7 @@ import eu.etaxonomy.cdm.io.common.mapping.CdmIoMapping; import eu.etaxonomy.cdm.io.common.mapping.CdmSingleAttributeMapperBase; import eu.etaxonomy.cdm.io.common.mapping.DbImportExtensionMapper; import eu.etaxonomy.cdm.io.common.mapping.DbImportMarkerMapper; +import eu.etaxonomy.cdm.io.common.mapping.DbSingleAttributeImportMapperBase; import eu.etaxonomy.cdm.model.agent.Team; import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase; import eu.etaxonomy.cdm.model.common.CdmBase; @@ -66,9 +64,7 @@ import eu.etaxonomy.cdm.model.common.IdentifiableSource; import eu.etaxonomy.cdm.model.common.Marker; import eu.etaxonomy.cdm.model.common.MarkerType; import eu.etaxonomy.cdm.model.reference.IArticle; -import eu.etaxonomy.cdm.model.reference.IBook; import eu.etaxonomy.cdm.model.reference.IBookSection; -import eu.etaxonomy.cdm.model.reference.IJournal; import eu.etaxonomy.cdm.model.reference.IPrintSeries; import eu.etaxonomy.cdm.model.reference.Reference; import eu.etaxonomy.cdm.model.reference.ReferenceFactory; @@ -98,13 +94,14 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { public BerlinModelReferenceImport(){ - super(); + super(dbTableName, pluralString); } protected void initializeMappers(BerlinModelImportState state){ for (CdmAttributeMapperBase mapper: classMappers){ - if (mapper instanceof DbImportExtensionMapper){ - ((DbImportExtensionMapper)mapper).initialize(state, Reference.class); + if (mapper instanceof DbSingleAttributeImportMapperBase){ + DbSingleAttributeImportMapperBase singleMapper = (DbSingleAttributeImportMapperBase)mapper; + singleMapper.initialize(state, Reference.class); } } return; @@ -165,108 +162,6 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { return null; //not needed } - -// @Override -// protected void doInvoke_old(BerlinModelImportState state){ -// logger.info("start make " + getPluralString() + " ..."); -// -// boolean success = true; -// initializeMappers(state); -// BerlinModelImportConfigurator config = state.getConfig(); -// Source source = config.getSource(); -// -// String strSelectId = " SELECT Reference.RefId as refId "; -// String strSelectFull = -// " SELECT Reference.* , InReference.RefId as InRefId, InReference.RefCategoryFk as InRefCategoryFk, " + -// " InInReference.RefId as InInRefId, InInReference.RefCategoryFk as InInRefCategoryFk, " + -// " InReference.InRefFk AS InRefInRefFk, InInReference.InRefFk AS InInRefInRefFk, RefSource.RefSource " ; -// String strFrom = " FROM Reference AS InInReference " + -// " RIGHT OUTER JOIN Reference AS InReference ON InInReference.RefId = InReference.InRefFk " + -// " RIGHT OUTER JOIN %s ON InReference.RefId = Reference.InRefFk " + -// " LEFT OUTER JOIN RefSource ON Reference.RefSourceFk = RefSource.RefSourceId " + -// " WHERE (1=1) "; -// String strWherePartitioned = " AND (Reference.refId IN ("+ ID_LIST_TOKEN + ") ) "; -// -// String referenceTable = CdmUtils.Nz(state.getConfig().getReferenceIdTable()); -// referenceTable = referenceTable.isEmpty() ? " Reference" : referenceTable + " as Reference "; -// String strIdFrom = String.format(strFrom, referenceTable ); -// -// -// //test max number of recursions -// String strQueryTestMaxRecursion = strSelectId + strIdFrom + -// " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NOT NULL) "; -// ResultSet testMaxRecursionResultSet = source.getResultSet(strQueryTestMaxRecursion); -// try { -// if (testMaxRecursionResultSet.next() == true){ -// logger.error("Maximum allowed InReference recursions exceeded in Berlin Model. Maximum recursion level is 2."); -// state.setUnsuccessfull(); -// return; -// } -// } catch (SQLException e1) { -// e1.printStackTrace(); -// logger.error("There are references with more then 2 in-reference recursion. Maximum number of allowed recursions is 2. Records will not be stored."); -// success = false; -// } -// -// String strSelectIdBase = strSelectId + strIdFrom; -// -// String referenceFilter = CdmUtils.Nz(state.getConfig().getReferenceIdTable()); -// if (! referenceFilter.isEmpty()){ -// referenceFilter = " AND " + referenceFilter + " "; -// } -// referenceFilter = ""; //don't use it for now -// -// String strIdQueryNoInRef = strSelectIdBase + -// " AND (Reference.InRefFk is NULL) " + referenceFilter; -// String strIdQuery1InRef = strSelectIdBase + -// " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NULL) " + referenceFilter; -// String strIdQuery2InRefs = strSelectIdBase + -// " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NULL) " + referenceFilter; -// -// if (config.getDoReferences() == CONCEPT_REFERENCES){ -// strIdQueryNoInRef += " AND ( Reference.refId IN ( SELECT ptRefFk FROM PTaxon) ) " + referenceFilter; -// } -// -// String strRecordQuery = strSelectFull + String.format(strFrom, " Reference ") + strWherePartitioned; -// -// int recordsPerTransaction = config.getRecordsPerTransaction(); -// try{ -// //NoInRefs -// ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQueryNoInRef, strRecordQuery, recordsPerTransaction); -// while (partitioner.nextPartition()){ -// partitioner.doPartition(this, state); -// } -// logger.info("end make references with no in-references ... " + getSuccessString(success)); -// -// if (config.getDoReferences() == ALL || config.getDoReferences() == NOMENCLATURAL){ -// -// //1InRef -// partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery1InRef, strRecordQuery, recordsPerTransaction); -// while (partitioner.nextPartition()){ -// partitioner.doPartition(this, state); -// } -// logger.info("end make references with no 1 in-reference ... " + getSuccessString(success)); -// -// //2InRefs -// partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery2InRefs, strRecordQuery, recordsPerTransaction); -// while (partitioner.nextPartition()){ -// partitioner.doPartition(this, state); -// } -// logger.info("end make references with no 2 in-reference ... " + getSuccessString(success)); -// } -// -// } catch (SQLException e) { -// logger.error("SQLException:" + e); -// state.setUnsuccessfull(); -// return; -// } -// logger.info("end make " + getPluralString() + " ... " + getSuccessString(success)); -// if (! success){ -// state.setUnsuccessfull(); -// } -// return; -// } - @Override protected void doInvoke(BerlinModelImportState state){ logger.info("start make " + getPluralString() + " ..."); @@ -289,23 +184,6 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { referenceTable = referenceTable.isEmpty() ? " Reference" : referenceTable + " as Reference "; String strIdFrom = String.format(strFrom, referenceTable ); - - //test max number of recursions -// String strQueryTestMaxRecursion = strSelectId + strIdFrom + -// " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NOT NULL) "; -// ResultSet testMaxRecursionResultSet = source.getResultSet(strQueryTestMaxRecursion); -// try { -// if (testMaxRecursionResultSet.next() == true){ -// logger.error("Maximum allowed InReference recursions exceeded in Berlin Model. Maximum recursion level is 2."); -// state.setUnsuccessfull(); -// return; -// } -// } catch (SQLException e1) { -// e1.printStackTrace(); -// logger.error("There are references with more then 2 in-reference recursion. Maximum number of allowed recursions is 2. Records will not be stored."); -// success = false; -// } - String strSelectIdBase = strSelectId + strIdFrom; String referenceFilter = CdmUtils.Nz(state.getConfig().getReferenceIdTable()); @@ -404,13 +282,13 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { //save and store in map logger.info("Save nomenclatural references (" + refCounter.nomRefCount + ")"); - getReferenceService().save(nomRefToSave.values()); + getReferenceService().saveOrUpdate(nomRefToSave.values()); logger.info("Save bibliographical references (" + refCounter.referenceCount +")"); - getReferenceService().save(biblioRefToSave.values()); + getReferenceService().saveOrUpdate(biblioRefToSave.values()); // }//end resultSetList - logger.info("end makeReferences ..." + getSuccessString(success));; +// logger.info("end makeReferences ..." + getSuccessString(success));; return success; } catch (SQLException e) { logger.error("SQLException:" + e); @@ -448,6 +326,7 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { Integer inRefFk = rs.getInt("inRefFk"); if (inRefFk != null){ + Reference thisNomRef = getReferenceOnlyFromMaps(relatedNomReferences, relatedBiblioReferences, String.valueOf(refId)); Reference thisBiblioRef = getReferenceOnlyFromMaps(relatedBiblioReferences, relatedNomReferences, String.valueOf(refId)); @@ -456,34 +335,45 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { boolean inRefExists = false; if (thisNomRef != null){ Reference inRef = (nomInReference != null)? nomInReference : biblioInReference; + if (inRef == null){ + logger.warn("No InRef found for nomRef: " + thisNomRef.getTitleCache() + "; RefId: " + refId + "; inRefFK: " + inRefFk); + } thisNomRef.setInReference(inRef); nomRefToSave.put(refId, thisNomRef); //remember that an in reference exists inRefExists |= (inRef != null); + thisNomRef.setTitleCache(null); + thisNomRef.getTitleCache(); } if (thisBiblioRef != null){ - Reference inRef = (biblioInReference != null)? biblioInReference : nomInReference; + Reference inRef = (biblioInReference != null)? biblioInReference : nomInReference ; + if (inRef == null){ + logger.warn("No InRef found for biblioRef: " + thisBiblioRef.getTitleCache() + "; RefId: " + refId + "; inRefFK: " + inRefFk); + } thisBiblioRef.setInReference(inRef); biblioRefToSave.put(refId, thisBiblioRef); //remember that an in reference exists inRefExists |= (inRef != null); + thisBiblioRef.setTitleCache(null); + thisBiblioRef.getTitleCache(); } if (inRefExists == false){ logger.warn("No in reference was saved though an 'inRefFk' is available. RefId " + refId); } + } } // end resultSet //save and store in map logger.info("Save nomenclatural references (" + refCounter.nomRefCount + ")"); - getReferenceService().save(nomRefToSave.values()); + getReferenceService().saveOrUpdate(nomRefToSave.values()); logger.info("Save bibliographical references (" + refCounter.referenceCount +")"); - getReferenceService().save(biblioRefToSave.values()); + getReferenceService().saveOrUpdate(biblioRefToSave.values()); // }//end resultSetList - logger.info("end makeReferences ..." + getSuccessString(success));; +// logger.info("end makeReferences ..." + getSuccessString(success));; return success; } catch (SQLException e) { logger.error("SQLException:" + e); @@ -616,8 +506,8 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { //idInSource String idInSource = (String)valueMap.get("IdInSource".toLowerCase()); - if (CdmUtils.isNotEmpty(idInSource)){ - IdentifiableSource source = IdentifiableSource.NewInstance(idInSource); + if (isNotBlank(idInSource)){ + IdentifiableSource source = IdentifiableSource.NewDataImportInstance(idInSource); source.setIdNamespace("import to Berlin Model"); referenceBase.addSource(source); } @@ -675,7 +565,7 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { boolean hasNomRef = false; boolean hasBiblioRef = false; - Reference sourceReference = state.getConfig().getSourceReference(); + Reference sourceReference = state.getTransactionalSourceReference(); //is Nomenclatural Reference if ( (CdmUtils.isNotEmpty(nomRefCache) && isPreliminary) || (CdmUtils.isNotEmpty(nomTitleAbbrev) && ! isPreliminary) ){ @@ -760,35 +650,12 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { Integer refId = (Integer)valueMap.get("refId".toLowerCase()); if (inRefFk != null){ - if (inRefCategoryFk == REF_JOURNAL){ - int inRefFkInt = (Integer)inRefFk; - if (true){ - //changed for first/second path implementation, if this is successful the following code can be deleted - }else if (existsInMapOrToSave(inRefFkInt, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences)){ - Reference inJournal = getReferenceFromMaps(inRefFkInt, nomRefToSave, relatedNomReferences); - if (inJournal == null){ - inJournal = getReferenceFromMaps(inRefFkInt, biblioRefToSave, relatedBiblioReferences); - logger.info("inJournal (" + inRefFkInt + ") found in referenceStore instead of nomRefStore."); - nomRefToSave.put(inRefFkInt, inJournal); - } - if (inJournal == null){ - logger.warn("inJournal for " + inRefFkInt + " is null. "+ - " InReference relation could not be set"); - //}else if (Reference.class.isAssignableFrom(inJournal.getClass())){ - }else if (inJournal.getType().equals(ReferenceType.Journal)){ - article.setInJournal((IJournal)inJournal); - }else{ - logger.warn("InJournal is not of type journal but of type " + inJournal.getType() + - " Inreference relation could not be set"); - } - }else{ - logger.error("Journal (refId = " + inRefFkInt + " ) for Article (refID = " + refId +") could not be found in nomRefStore. Inconsistency error. "); - //success = false;; - } - }else{ + if (inRefCategoryFk != REF_JOURNAL){ logger.warn("Wrong inrefCategory for Article (refID = " + refId +"). Type must be 'Journal' but was not (RefCategoryFk=" + inRefCategoryFk + "))." + - " InReference was not added to Article! "); + " InReference was added anyway! "); } + }else{ + logger.warn ("Article has no inreference: " + refId); } makeStandardMapper(valueMap, (Reference)article); //url, pages, series, volume return (Reference)article; @@ -804,40 +671,18 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { //null -> error logger.warn("Part-Of-Other-Title has no inRefCategoryFk! RefId = " + refId + ". ReferenceType set to Generic."); result = makeUnknown(valueMap); + }else if (inRefFk == null){ + logger.warn("Part-Of-Other-Title has in in reference: " + refId); + result = makeUnknown(valueMap); }else if (inRefCategoryFk == REF_BOOK){ //BookSection IBookSection bookSection = ReferenceFactory.newBookSection(); result = (Reference)bookSection; - if (inRefFk != null && false){ //&& false added for first/second path implementation, following code can be deleted or moved if this is successful - int inRefFkInt = (Integer) inRefFk; - if (existsInMapOrToSave(inRefFkInt, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences)){ - Reference inBook = getReferenceFromMaps(inRefFkInt, nomRefToSave, relatedNomReferences); - if (inBook == null){ - inBook = getReferenceFromMaps(inRefFkInt, biblioRefToSave, relatedBiblioReferences); - logger.info("inBook (" + inRefFkInt + ") found in referenceStore instead of nomRefStore."); - nomRefToSave.put(inRefFkInt, inBook); - } - if (inBook == null){ - logger.warn("inBook for " + inRefFkInt + " is null. "+ - " InReference relation could not be set");; - //}else if (Book.class.isAssignableFrom(inBook.getClass())){ - }else if (inBook.getType().equals(ReferenceType.Book)){ - bookSection.setInBook((IBook)inBook); - //TODO - }else{ - logger.warn("InBook is not of type book but of type " + inBook.getClass().getSimpleName() + - " Inreference relation could not be set"); - } - }else{ - logger.error("Book (refId = " + inRefFkInt + ") for part_of_other_title (refID = " + refId +") could not be found in nomRefStore. Inconsistency error. "); - //success = false; - } - } }else if (inRefCategoryFk == REF_ARTICLE){ //Article //TODO - logger.warn("Reference (refId = " + refId + ") of type 'part_of_other_title' is part of 'article'." + - " This type is not implemented yet. Generic reference created instead") ; + logger.info("Reference (refId = " + refId + ") of type 'part_of_other_title' is part of 'article'." + + " There is no specific reference type for such in references yet. Generic reference created instead") ; result = ReferenceFactory.newGeneric(); }else if (inRefCategoryFk == REF_JOURNAL){ //TODO @@ -1183,23 +1028,7 @@ public class BerlinModelReferenceImport extends BerlinModelImportBase { BerlinModelReferenceImportValidator validator = new BerlinModelReferenceImportValidator(); return validator.validate(state, this); } - - - /* (non-Javadoc) - * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName() - */ - @Override - protected String getTableName() { - return dbTableName; - } - /* (non-Javadoc) - * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString() - */ - @Override - public String getPluralString() { - return pluralString; - } /* (non-Javadoc) * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)