Revision 1594c653
Added by Andreas Müller over 2 years ago
| cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java | ||
|---|---|---|
| 53 | 53 |
|
| 54 | 54 |
private boolean isCheck; |
| 55 | 55 |
|
| 56 |
private ImportDeduplicationHelper<ImportStateBase<CONFIG,?>> deduplicationHelper;
|
|
| 56 |
private ImportDeduplicationHelper deduplicationHelper; |
|
| 57 | 57 |
|
| 58 | 58 |
private Map<Object,Classification> treeMap = new HashMap<>(); |
| 59 | 59 |
|
| ... | ... | |
| 402 | 402 |
return null; |
| 403 | 403 |
} |
| 404 | 404 |
|
| 405 |
public ImportDeduplicationHelper<ImportStateBase<CONFIG,?>> getDeduplicationHelper() {
|
|
| 405 |
public ImportDeduplicationHelper getDeduplicationHelper() {
|
|
| 406 | 406 |
return deduplicationHelper; |
| 407 | 407 |
} |
| 408 |
public void setDeduplicationHelper(ImportDeduplicationHelper<ImportStateBase<CONFIG,?>> deduplicationHelper) {
|
|
| 408 |
public void setDeduplicationHelper(ImportDeduplicationHelper deduplicationHelper) {
|
|
| 409 | 409 |
this.deduplicationHelper = deduplicationHelper; |
| 410 | 410 |
} |
| 411 | 411 |
|
| 412 | 412 |
@Override |
| 413 | 413 |
public void setCurrentIO(IO currentIO) {
|
| 414 | 414 |
super.setCurrentIO(currentIO); |
| 415 |
this.deduplicationHelper.reset(); |
|
| 415 |
if (this.deduplicationHelper != null){
|
|
| 416 |
this.deduplicationHelper.reset(); |
|
| 417 |
} |
|
| 416 | 418 |
this.deduplicationHelper = ImportDeduplicationHelper.NewInstance(currentIO, this); |
| 417 | 419 |
} |
| 418 | 420 |
} |
| cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/utils/ImportDeduplicationHelper.java | ||
|---|---|---|
| 52 | 52 |
* @author a.mueller |
| 53 | 53 |
* @since 11.02.2017 |
| 54 | 54 |
*/ |
| 55 |
public class ImportDeduplicationHelper<STATE extends ImportStateBase> {
|
|
| 55 |
public class ImportDeduplicationHelper {
|
|
| 56 |
|
|
| 56 | 57 |
private static final Logger logger = Logger.getLogger(ImportDeduplicationHelper.class); |
| 57 | 58 |
|
| 58 | 59 |
private ICdmRepository repository; |
| 59 | 60 |
|
| 60 |
private STATE state;
|
|
| 61 |
private ImportStateBase<?,?> state;
|
|
| 61 | 62 |
|
| 62 | 63 |
boolean referenceMapIsInitialized = false; |
| 63 | 64 |
boolean nameMapIsInitialized = false; |
| ... | ... | |
| 99 | 100 |
|
| 100 | 101 |
/** |
| 101 | 102 |
* @param repository |
| 102 |
* @param state not used, only for correct casting of generics
|
|
| 103 |
* @param state |
|
| 103 | 104 |
* @return |
| 104 | 105 |
*/ |
| 105 |
public static <STATE extends ImportStateBase<?,?>> ImportDeduplicationHelper<STATE> NewInstance(ICdmRepository repository, STATE state){
|
|
| 106 |
return new ImportDeduplicationHelper<>(repository, state);
|
|
| 106 |
public static <STATE extends ImportStateBase<?,?>> ImportDeduplicationHelper NewInstance(ICdmRepository repository, STATE state){
|
|
| 107 |
return new ImportDeduplicationHelper(repository, state); |
|
| 107 | 108 |
} |
| 108 | 109 |
|
| 109 | 110 |
// ************************ CONSTRUCTOR *****************************/ |
| 110 | 111 |
|
| 111 |
public ImportDeduplicationHelper(ICdmRepository repository, STATE state) {
|
|
| 112 |
private ImportDeduplicationHelper(ICdmRepository repository, ImportStateBase<?,?> state) {
|
|
| 112 | 113 |
this.repository = repository; |
| 113 | 114 |
if (repository == null){
|
| 114 | 115 |
logger.warn("Repository is null. Deduplication does not work against database.");
|
| ... | ... | |
| 384 | 385 |
* @param state the import state |
| 385 | 386 |
* @param name the name with authors and references to replace |
| 386 | 387 |
*/ |
| 387 |
public void replaceAuthorNamesAndNomRef(STATE state,
|
|
| 388 |
INonViralName name) {
|
|
| 388 |
public void replaceAuthorNamesAndNomRef(INonViralName name) {
|
|
| 389 |
|
|
| 389 | 390 |
TeamOrPersonBase<?> combAuthor = name.getCombinationAuthorship(); |
| 390 |
name.setCombinationAuthorship(getExistingAuthor(state, combAuthor));
|
|
| 391 |
name.setCombinationAuthorship(getExistingAuthor(combAuthor)); |
|
| 391 | 392 |
|
| 392 | 393 |
TeamOrPersonBase<?> exAuthor = name.getExCombinationAuthorship(); |
| 393 |
name.setExCombinationAuthorship(getExistingAuthor(state, exAuthor));
|
|
| 394 |
name.setExCombinationAuthorship(getExistingAuthor(exAuthor)); |
|
| 394 | 395 |
|
| 395 | 396 |
TeamOrPersonBase<?> basioAuthor = name.getBasionymAuthorship(); |
| 396 |
name.setBasionymAuthorship(getExistingAuthor(state, basioAuthor));
|
|
| 397 |
name.setBasionymAuthorship(getExistingAuthor(basioAuthor)); |
|
| 397 | 398 |
|
| 398 | 399 |
TeamOrPersonBase<?> exBasioAuthor = name.getExBasionymAuthorship(); |
| 399 |
name.setExBasionymAuthorship(getExistingAuthor(state, exBasioAuthor));
|
|
| 400 |
name.setExBasionymAuthorship(getExistingAuthor(exBasioAuthor)); |
|
| 400 | 401 |
|
| 401 | 402 |
INomenclaturalReference nomRef = name.getNomenclaturalReference(); |
| 402 | 403 |
if (nomRef != null){
|
| 403 | 404 |
TeamOrPersonBase<?> refAuthor = nomRef.getAuthorship(); |
| 404 |
nomRef.setAuthorship(getExistingAuthor(state, refAuthor));
|
|
| 405 |
nomRef.setAuthorship(getExistingAuthor(refAuthor)); |
|
| 405 | 406 |
|
| 406 |
Reference existingRef = getExistingReference(state, (Reference)nomRef);
|
|
| 407 |
Reference existingRef = getExistingReference((Reference)nomRef); |
|
| 407 | 408 |
if (existingRef != null){
|
| 408 | 409 |
name.setNomenclaturalReference(existingRef); |
| 409 | 410 |
} |
| 410 | 411 |
} |
| 411 | 412 |
} |
| 412 | 413 |
|
| 413 |
public <T extends TeamOrPersonBase<?>> T getExistingAuthor(STATE state, |
|
| 414 |
T author) {
|
|
| 414 |
public <T extends TeamOrPersonBase<?>> T getExistingAuthor(T author) {
|
|
| 415 | 415 |
if (author == null){
|
| 416 | 416 |
return null; |
| 417 | 417 |
}else{
|
| 418 |
initAgentMap(state);
|
|
| 418 |
initAgentMap(); |
|
| 419 | 419 |
initAuthorTitleCaches(author); |
| 420 | 420 |
T result = getTeamOrPerson(author); |
| 421 | 421 |
if (result == null){
|
| 422 | 422 |
putAgentBase(author.getTitleCache(), author); |
| 423 | 423 |
if (author.isInstanceOf(Team.class)){
|
| 424 |
handleTeam(state, CdmBase.deproxy(author, Team.class));
|
|
| 424 |
handleTeam(CdmBase.deproxy(author, Team.class)); |
|
| 425 | 425 |
} |
| 426 | 426 |
result = author; |
| 427 | 427 |
} |
| ... | ... | |
| 454 | 454 |
ref.getTitleCache(); |
| 455 | 455 |
} |
| 456 | 456 |
|
| 457 |
public AgentBase<?> getExistingAgent(STATE state, |
|
| 458 |
AgentBase<?> agent) {
|
|
| 457 |
public AgentBase<?> getExistingAgent(AgentBase<?> agent) {
|
|
| 459 | 458 |
if (agent == null){
|
| 460 | 459 |
return null; |
| 461 | 460 |
} else if (agent.isInstanceOf(TeamOrPersonBase.class)){
|
| 462 |
return getExistingAuthor(state, CdmBase.deproxy(agent, TeamOrPersonBase.class));
|
|
| 461 |
return getExistingAuthor(CdmBase.deproxy(agent, TeamOrPersonBase.class)); |
|
| 463 | 462 |
}else{
|
| 464 |
initAgentMap(state);
|
|
| 463 |
initAgentMap(); |
|
| 465 | 464 |
Institution result = institutionMap.get(agent.getTitleCache()); |
| 466 | 465 |
if (result == null){
|
| 467 | 466 |
putAgentBase(agent.getTitleCache(), agent); |
| ... | ... | |
| 472 | 471 |
} |
| 473 | 472 |
|
| 474 | 473 |
@SuppressWarnings("rawtypes")
|
| 475 |
private void initAgentMap(STATE state) {
|
|
| 474 |
private void initAgentMap() {
|
|
| 476 | 475 |
if (!agentMapIsInitialized && repository != null){
|
| 477 | 476 |
List<String> propertyPaths = Arrays.asList("");
|
| 478 | 477 |
List<AgentBase> existingAgents = repository.getAgentService().list(null, null, null, null, propertyPaths); |
| ... | ... | |
| 483 | 482 |
} |
| 484 | 483 |
} |
| 485 | 484 |
|
| 486 |
private void handleTeam(STATE state, Team team) {
|
|
| 485 |
private void handleTeam(Team team) {
|
|
| 487 | 486 |
List<Person> members = team.getTeamMembers(); |
| 488 | 487 |
for (int i =0; i< members.size(); i++){
|
| 489 | 488 |
Person person = CdmBase.deproxy(members.get(i)); |
| ... | ... | |
| 496 | 495 |
} |
| 497 | 496 |
} |
| 498 | 497 |
|
| 499 |
public Collection getExistingCollection(STATE state, Collection collection) {
|
|
| 498 |
public Collection getExistingCollection(Collection collection) {
|
|
| 500 | 499 |
if (collection == null){
|
| 501 | 500 |
return null; |
| 502 | 501 |
}else{
|
| 503 |
initCollectionMap(state);
|
|
| 502 |
initCollectionMap(); |
|
| 504 | 503 |
Collection result = getMatchingCollections(collection).orElse(null); |
| 505 | 504 |
if (result == null){
|
| 506 | 505 |
result = collection; |
| ... | ... | |
| 514 | 513 |
} |
| 515 | 514 |
} |
| 516 | 515 |
|
| 517 |
private void initCollectionMap(STATE state) {
|
|
| 516 |
private void initCollectionMap() {
|
|
| 518 | 517 |
if (!collectionMapIsInitialized && repository != null){
|
| 519 | 518 |
List<String> propertyPaths = Arrays.asList("");
|
| 520 | 519 |
List<Collection> existingCollections = repository.getCollectionService().list(null, null, null, null, propertyPaths); |
| ... | ... | |
| 525 | 524 |
} |
| 526 | 525 |
} |
| 527 | 526 |
|
| 528 |
public Reference getExistingReference(STATE state, Reference ref) {
|
|
| 527 |
public Reference getExistingReference(Reference ref) {
|
|
| 529 | 528 |
if (ref == null){
|
| 530 | 529 |
return null; |
| 531 | 530 |
}else{
|
| 532 |
initRerenceMap(state);
|
|
| 531 |
initRerenceMap(); |
|
| 533 | 532 |
initReferenceCaches(ref); |
| 534 | 533 |
Reference result = getMatchingReference(ref).orElse(null); |
| 535 | 534 |
if (result == null){
|
| 536 | 535 |
result = ref; |
| 537 | 536 |
Reference inRef = result.getInReference(); |
| 538 | 537 |
if (inRef != null){
|
| 539 |
result.setInReference(getExistingReference(state, result.getInReference()));
|
|
| 538 |
result.setInReference(getExistingReference(result.getInReference())); |
|
| 540 | 539 |
} |
| 541 | 540 |
putReference(result.getTitleCache(), result); |
| 542 | 541 |
}else{
|
| ... | ... | |
| 548 | 547 |
} |
| 549 | 548 |
} |
| 550 | 549 |
|
| 551 |
private void initRerenceMap(STATE state) {
|
|
| 550 |
private void initRerenceMap() {
|
|
| 552 | 551 |
if (!referenceMapIsInitialized && repository != null){
|
| 553 | 552 |
List<String> propertyPaths = Arrays.asList("");
|
| 554 | 553 |
List<Reference> existingReferences = repository.getReferenceService().list(null, null, null, null, propertyPaths); |
| ... | ... | |
| 559 | 558 |
} |
| 560 | 559 |
} |
| 561 | 560 |
|
| 562 |
public <NAME extends INonViralName> NAME getExistingName(STATE state, NAME name) {
|
|
| 561 |
public <NAME extends INonViralName> NAME getExistingName(NAME name) {
|
|
| 563 | 562 |
if (name == null){
|
| 564 | 563 |
return null; |
| 565 | 564 |
}else{
|
| 566 |
initNameMap(state);
|
|
| 565 |
initNameMap(); |
|
| 567 | 566 |
@SuppressWarnings("unchecked")
|
| 568 | 567 |
NAME result = (NAME)getMatchingName(name).orElse(null); |
| 569 | 568 |
if (result == null){
|
| ... | ... | |
| 572 | 571 |
for (HybridRelationship rel : parentRelations){
|
| 573 | 572 |
INonViralName parent = rel.getParentName(); |
| 574 | 573 |
if (parent != null){
|
| 575 |
rel.setParentName(getExistingName(state, parent));
|
|
| 574 |
rel.setParentName(getExistingName(parent)); |
|
| 576 | 575 |
} |
| 577 | 576 |
} |
| 578 | 577 |
putName(result.getTitleCache(), result); |
| ... | ... | |
| 585 | 584 |
} |
| 586 | 585 |
} |
| 587 | 586 |
|
| 588 |
private void initNameMap(STATE state) {
|
|
| 587 |
private void initNameMap() {
|
|
| 589 | 588 |
if (!nameMapIsInitialized && repository != null){
|
| 590 | 589 |
List<String> propertyPaths = Arrays.asList("");
|
| 591 | 590 |
List<TaxonName> existingNames = repository.getNameService().list(null, null, null, null, propertyPaths); |
| ... | ... | |
| 596 | 595 |
} |
| 597 | 596 |
} |
| 598 | 597 |
|
| 599 |
public Rights getExistingCopyright(STATE state, |
|
| 600 |
Rights right) {
|
|
| 598 |
public Rights getExistingCopyright(Rights right) {
|
|
| 601 | 599 |
if (right == null || !RightsType.COPYRIGHT().equals(right.getType())){
|
| 602 | 600 |
return null; |
| 603 | 601 |
}else{
|
| 604 |
initCopyrightMap(state);
|
|
| 602 |
initCopyrightMap(); |
|
| 605 | 603 |
String key = makeCopyrightKey(right); |
| 606 | 604 |
Set<Rights> set = copyrightMap.get(key); |
| 607 | 605 |
if (set == null || set.isEmpty()){
|
| ... | ... | |
| 615 | 613 |
} |
| 616 | 614 |
} |
| 617 | 615 |
|
| 618 |
private void initCopyrightMap(STATE state) {
|
|
| 616 |
private void initCopyrightMap() {
|
|
| 619 | 617 |
if (!copyrightMapIsInitialized && repository != null){
|
| 620 | 618 |
List<String> propertyPaths = Arrays.asList("");
|
| 621 | 619 |
List<Rights> existingRights = repository.getRightsService().list(null, null, null, null, propertyPaths); |
| cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupImportBase.java | ||
|---|---|---|
| 1152 | 1152 |
NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance(); |
| 1153 | 1153 |
protected TeamOrPersonBase<?> createAuthor(MarkupImportState state, String authorTitle) {
|
| 1154 | 1154 |
TeamOrPersonBase<?> result = parser.author(authorTitle); |
| 1155 |
return state.getDeduplicationHelper().getExistingAuthor(state, result);
|
|
| 1155 |
return state.getDeduplicationHelper().getExistingAuthor(result); |
|
| 1156 | 1156 |
} |
| 1157 | 1157 |
|
| 1158 | 1158 |
protected String getAndRemoveMapKey(Map<String, String> map, String key) {
|
| ... | ... | |
| 2100 | 2100 |
String details = refMap.get(DETAILS); |
| 2101 | 2101 |
// String label = makeLabel(state, refMap, next); |
| 2102 | 2102 |
Reference ref = createReference(state, refMap, next); |
| 2103 |
ref = state.getDeduplicationHelper().getExistingReference(state, ref);
|
|
| 2103 |
ref = state.getDeduplicationHelper().getExistingReference(ref); |
|
| 2104 | 2104 |
|
| 2105 | 2105 |
String label2 = ref.getTitleCache(); //TODO preliminary for debugging and testing |
| 2106 | 2106 |
result.content.add(new LabeledReference(ref, details, label)); |
| cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupNomenclatureImport.java | ||
|---|---|---|
| 242 | 242 |
fillName(state, nameMap, name, misappliedRelation, next); |
| 243 | 243 |
} |
| 244 | 244 |
handleNomText(state, parentEvent, text, isNameType); |
| 245 |
state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(state, name);
|
|
| 245 |
state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(name); |
|
| 246 | 246 |
handleNameStatus(state, name, next); |
| 247 | 247 |
state.setNameStatus(null); |
| 248 | 248 |
return name; |
| ... | ... | |
| 663 | 663 |
String classValue, TaxonRelationship misappliedRel, |
| 664 | 664 |
Reference reference, String microCitation, |
| 665 | 665 |
XMLEvent parentEvent) {
|
| 666 |
reference = state.getDeduplicationHelper().getExistingReference(state, reference);
|
|
| 666 |
reference = state.getDeduplicationHelper().getExistingReference(reference); |
|
| 667 | 667 |
if (misappliedRel != null){
|
| 668 | 668 |
if (!PUBLICATION.equalsIgnoreCase(classValue)){
|
| 669 | 669 |
fireWarningEvent("'Usage' not handled correctly for misidentifications", parentEvent, 4);
|
| cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/media/in/MediaExcelImport.java | ||
|---|---|---|
| 23 | 23 |
|
| 24 | 24 |
import eu.etaxonomy.cdm.common.URI; |
| 25 | 25 |
import eu.etaxonomy.cdm.common.media.CdmImageInfo; |
| 26 |
import eu.etaxonomy.cdm.io.common.utils.ImportDeduplicationHelper; |
|
| 27 | 26 |
import eu.etaxonomy.cdm.io.excel.common.ExcelImportBase; |
| 28 | 27 |
import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase; |
| 29 | 28 |
import eu.etaxonomy.cdm.io.media.in.MediaExcelImportConfigurator.MediaTitleEnum; |
| ... | ... | |
| 63 | 62 |
private static final String COL_ARTIST = "artist"; |
| 64 | 63 |
private static final String COL_DATE = "date"; |
| 65 | 64 |
|
| 66 |
private ImportDeduplicationHelper<MediaExcelImportState> deduplicationHelper; |
|
| 67 |
|
|
| 68 | 65 |
@Override |
| 69 | 66 |
protected void analyzeRecord(Map<String, String> record, MediaExcelImportState state) {
|
| 70 | 67 |
// do nothing |
| ... | ... | |
| 112 | 109 |
if (isNotBlank(copyright)){
|
| 113 | 110 |
AgentBase<?> agent = makePerson(state, copyright, line); |
| 114 | 111 |
Rights right = Rights.NewInstance(RightsType.COPYRIGHT(), agent); |
| 115 |
right = getDeduplicationHelper(state).getExistingCopyright(state, right);
|
|
| 112 |
right = state.getDeduplicationHelper().getExistingCopyright(right);
|
|
| 116 | 113 |
media.addRights(right); |
| 117 | 114 |
} |
| 118 | 115 |
|
| ... | ... | |
| 265 | 262 |
return list; |
| 266 | 263 |
} |
| 267 | 264 |
|
| 268 |
private ImportDeduplicationHelper<MediaExcelImportState> getDeduplicationHelper(MediaExcelImportState state) {
|
|
| 269 |
if (this.deduplicationHelper == null){
|
|
| 270 |
this.deduplicationHelper = ImportDeduplicationHelper.NewInstance(this, state); |
|
| 271 |
} |
|
| 272 |
return deduplicationHelper; |
|
| 273 |
} |
|
| 274 |
|
|
| 275 | 265 |
private Person makePerson(MediaExcelImportState state, String artist, String line) {
|
| 276 | 266 |
Person person = Person.NewInstance(); |
| 277 | 267 |
artist = artist.trim(); |
| ... | ... | |
| 296 | 286 |
|
| 297 | 287 |
} |
| 298 | 288 |
|
| 299 |
Person result = getDeduplicationHelper(state).getExistingAuthor(null, person);
|
|
| 289 |
Person result = state.getDeduplicationHelper().getExistingAuthor(person);
|
|
| 300 | 290 |
return result; |
| 301 | 291 |
} |
| 302 | 292 |
|
| cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/tropicos/in/TropicosNameImport.java | ||
|---|---|---|
| 109 | 109 |
if (record.get(OUTPUT_COLLATION) != null){
|
| 110 | 110 |
//ignore |
| 111 | 111 |
} |
| 112 |
state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(state, name);
|
|
| 112 |
state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(name); |
|
| 113 | 113 |
|
| 114 | 114 |
getNameService().saveOrUpdate(name); |
| 115 | 115 |
state.getResult().addNewRecords(TaxonName.class.getSimpleName(), 1); |
| cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/wfo/in/WfoAccessTaxonImport.java | ||
|---|---|---|
| 100 | 100 |
testAlwaysEmptyFields(state); |
| 101 | 101 |
|
| 102 | 102 |
|
| 103 |
state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(state, name);
|
|
| 103 |
state.getDeduplicationHelper().replaceAuthorNamesAndNomRef(name); |
|
| 104 | 104 |
|
| 105 | 105 |
if (!name.isPersited()){
|
| 106 | 106 |
state.getResult().addNewRecords(TaxonName.class.getSimpleName(), 1); |
Also available in: Unified diff
ref #9890 move importDeduplicationHelper to import state and include state in helper (cont.)