2 * Copyright (C) 2007 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
10 package eu
.etaxonomy
.cdm
.io
.berlinModel
.in
;
12 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_ARTICLE
;
13 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_BOOK
;
14 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_CONFERENCE_PROCEEDINGS
;
15 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_DATABASE
;
16 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_INFORMAL
;
17 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_JOURNAL
;
18 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_JOURNAL_VOLUME
;
19 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_PART_OF_OTHER_TITLE
;
20 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_PRINT_SERIES
;
21 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_UNKNOWN
;
22 import static eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
.REF_WEBSITE
;
23 import static eu
.etaxonomy
.cdm
.io
.common
.ImportHelper
.NO_OVERWRITE
;
24 import static eu
.etaxonomy
.cdm
.io
.common
.ImportHelper
.OBLIGATORY
;
25 import static eu
.etaxonomy
.cdm
.io
.common
.ImportHelper
.OVERWRITE
;
27 import java
.net
.URISyntaxException
;
28 import java
.sql
.ResultSet
;
29 import java
.sql
.SQLException
;
30 import java
.util
.ArrayList
;
31 import java
.util
.Arrays
;
32 import java
.util
.HashMap
;
33 import java
.util
.HashSet
;
34 import java
.util
.List
;
37 import java
.util
.UUID
;
38 import java
.util
.regex
.Matcher
;
39 import java
.util
.regex
.Pattern
;
41 import org
.apache
.log4j
.Logger
;
42 import org
.springframework
.stereotype
.Component
;
44 import eu
.etaxonomy
.cdm
.common
.CdmUtils
;
45 import eu
.etaxonomy
.cdm
.common
.DOI
;
46 import eu
.etaxonomy
.cdm
.common
.URI
;
47 import eu
.etaxonomy
.cdm
.io
.berlinModel
.BerlinModelTransformer
;
48 import eu
.etaxonomy
.cdm
.io
.berlinModel
.in
.validation
.BerlinModelReferenceImportValidator
;
49 import eu
.etaxonomy
.cdm
.io
.common
.ICdmImport
;
50 import eu
.etaxonomy
.cdm
.io
.common
.IImportConfigurator
;
51 import eu
.etaxonomy
.cdm
.io
.common
.ImportHelper
;
52 import eu
.etaxonomy
.cdm
.io
.common
.ResultSetPartitioner
;
53 import eu
.etaxonomy
.cdm
.io
.common
.Source
;
54 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.CdmAttributeMapperBase
;
55 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.CdmIoMapping
;
56 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.CdmSingleAttributeMapperBase
;
57 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.DbImportExtensionMapper
;
58 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.DbImportMarkerMapper
;
59 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.DbSingleAttributeImportMapperBase
;
60 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.berlinModel
.CdmOneToManyMapper
;
61 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.berlinModel
.CdmStringMapper
;
62 import eu
.etaxonomy
.cdm
.io
.common
.mapping
.berlinModel
.CdmUriMapper
;
63 import eu
.etaxonomy
.cdm
.io
.common
.utils
.ImportDeduplicationHelper
;
64 import eu
.etaxonomy
.cdm
.model
.agent
.Person
;
65 import eu
.etaxonomy
.cdm
.model
.agent
.Team
;
66 import eu
.etaxonomy
.cdm
.model
.agent
.TeamOrPersonBase
;
67 import eu
.etaxonomy
.cdm
.model
.common
.CdmBase
;
68 import eu
.etaxonomy
.cdm
.model
.common
.ExtensionType
;
69 import eu
.etaxonomy
.cdm
.model
.common
.IdentifiableSource
;
70 import eu
.etaxonomy
.cdm
.model
.common
.Identifier
;
71 import eu
.etaxonomy
.cdm
.model
.common
.Marker
;
72 import eu
.etaxonomy
.cdm
.model
.common
.MarkerType
;
73 import eu
.etaxonomy
.cdm
.model
.reference
.IArticle
;
74 import eu
.etaxonomy
.cdm
.model
.reference
.IBookSection
;
75 import eu
.etaxonomy
.cdm
.model
.reference
.IPrintSeries
;
76 import eu
.etaxonomy
.cdm
.model
.reference
.Reference
;
77 import eu
.etaxonomy
.cdm
.model
.reference
.ReferenceFactory
;
78 import eu
.etaxonomy
.cdm
.model
.reference
.ReferenceType
;
79 import eu
.etaxonomy
.cdm
.model
.term
.DefinedTerm
;
80 import eu
.etaxonomy
.cdm
.model
.term
.TermVocabulary
;
81 import eu
.etaxonomy
.cdm
.strategy
.cache
.agent
.PersonDefaultCacheStrategy
;
82 import eu
.etaxonomy
.cdm
.strategy
.cache
.agent
.TeamDefaultCacheStrategy
;
89 public class BerlinModelReferenceImport
extends BerlinModelImportBase
{
90 private static final long serialVersionUID
= -3667566958769967591L;
92 private static final Logger logger
= Logger
.getLogger(BerlinModelReferenceImport
.class);
94 public static final String REFERENCE_NAMESPACE
= "Reference";
95 private static final String REF_AUTHOR_NAMESPACE
= "Reference.refAuthorString";
98 public static final UUID REF_DEPOSITED_AT_UUID
= UUID
.fromString("23ca88c7-ce73-41b2-8ca3-2cb22f013beb");
99 public static final UUID REF_SOURCE_UUID
= UUID
.fromString("d6432582-2216-4b08-b0db-76f6c1013141");
100 public static final UUID DATE_STRING_UUID
= UUID
.fromString("e4130eae-606e-4b0c-be4f-e93dc161be7d");
101 public static final UUID IS_PAPER_UUID
= UUID
.fromString("8a326129-d0d0-4f9d-bbdf-8d86b037c65e");
103 private static ImportDeduplicationHelper
<BerlinModelImportState
> deduplicationHelper
;
105 private final int modCount
= 1000;
106 private static final String pluralString
= "references";
107 private static final String dbTableName
= "reference";
110 public BerlinModelReferenceImport(){
111 super(dbTableName
, pluralString
);
114 protected void initializeMappers(BerlinModelImportState state
){
115 for (CdmAttributeMapperBase mapper
: classMappers
){
116 if (mapper
instanceof DbSingleAttributeImportMapperBase
){
117 @SuppressWarnings("unchecked")
118 DbSingleAttributeImportMapperBase
<BerlinModelImportState
,Reference
> singleMapper
=
119 (DbSingleAttributeImportMapperBase
<BerlinModelImportState
,Reference
>)mapper
;
120 singleMapper
.initialize(state
, Reference
.class);
126 private Set
<Integer
> commonNameRefSet
= null;
127 private void initializeCommonNameRefMap(BerlinModelImportState state
) throws SQLException
{
128 if (state
.getConfig().isEuroMed()){
129 commonNameRefSet
= new HashSet
<>();
130 String queryStr
= "SELECT DISTINCT RefFk "
131 + " FROM emCommonName ";
132 ResultSet rs
= state
.getConfig().getSource().getResultSet(queryStr
);
134 commonNameRefSet
.add(rs
.getInt("RefFk"));
139 protected static CdmAttributeMapperBase
[] classMappers
= new CdmAttributeMapperBase
[]{
140 new CdmStringMapper("edition", "edition"),
141 new CdmStringMapper("volume", "volume"),
142 new CdmStringMapper("publisher", "publisher"),
143 new CdmStringMapper("publicationTown", "placePublished"),
144 new CdmStringMapper("isbn", "isbn"),
145 new CdmStringMapper("isbn", "isbn"),
146 new CdmStringMapper("pageString", "pages"),
147 new CdmStringMapper("series", "seriesPart"),
148 new CdmStringMapper("issn", "issn"),
149 new CdmUriMapper("url", "uri"),
150 DbImportExtensionMapper
.NewInstance("NomStandard", ExtensionType
.NOMENCLATURAL_STANDARD()),
151 DbImportExtensionMapper
.NewInstance("DateString", DATE_STRING_UUID
, "Date String", "Date String", "dates"),
152 DbImportExtensionMapper
.NewInstance("RefDepositedAt", REF_DEPOSITED_AT_UUID
, "Ref. deposited at", "Reference is deposited at", "at"),
153 DbImportExtensionMapper
.NewInstance("RefSource", REF_SOURCE_UUID
, "RefSource", "Reference Source", "source"),
154 DbImportMarkerMapper
.NewInstance("isPaper", IS_PAPER_UUID
, "is paper", "is paper", "paper", false)
155 //not yet supported by model
156 ,new CdmStringMapper("refAuthorString", "refAuthorString"),
160 protected static String
[] operationalAttributes
= new String
[]{
161 "refId", "refCache", "nomRefCache", "preliminaryFlag", "inRefFk", "title", "nomTitleAbbrev",
162 "refAuthorString", "nomAuthorTeamFk",
163 "refCategoryFk", "thesisFlag", "informalRefCategory", "idInSource"
166 protected static String
[] createdAndNotesAttributes
= new String
[]{
167 "created_When", "updated_When", "created_Who", "updated_Who", "notes"
170 protected static String
[] unclearMappers
= new String
[]{
171 /*"isPaper",*/ "exportDate",
179 //type to count the references nomReferences that have been created and saved
180 private class RefCounter
{
181 RefCounter() {refCount
= 0;}
186 public String
toString(){return String
.valueOf(refCount
) + "/" + String
.valueOf(dedupCount
) ;}
190 protected String
getRecordQuery(BerlinModelImportConfigurator config
) {
191 return null; //not needed
195 protected void doInvoke(BerlinModelImportState state
){
196 logger
.info("start make " + getPluralString() + " ...");
197 deduplicationHelper
= ImportDeduplicationHelper
.NewInstance(this, state
);
199 boolean success
= true;
200 initializeMappers(state
);
202 initializeCommonNameRefMap(state
);
203 } catch (SQLException e1
) {
204 e1
.printStackTrace();
205 logger
.error("Error in initializeCommonNameRefMap in BerlinModelReferenceimport");
207 BerlinModelImportConfigurator config
= state
.getConfig();
208 Source source
= config
.getSource();
210 String strSelectId
= " SELECT Reference.RefId as refId ";
211 String strSelectFull
=
212 " SELECT Reference.* ,InReference.RefCategoryFk as InRefCategoryFk, RefSource.RefSource " ;
215 " LEFT OUTER JOIN Reference as InReference ON InReference.refId = Reference.inRefFk " +
216 " LEFT OUTER JOIN RefSource ON Reference.RefSourceFk = RefSource.RefSourceId " +
218 String strOrderBy
= " ORDER BY InReference.inRefFk, Reference.inRefFk "; //to make in-references available in first run
219 String strWherePartitioned
= " AND (Reference.refId IN ("+ ID_LIST_TOKEN
+ ") ) ";
221 String referenceTable
= CdmUtils
.Nz(state
.getConfig().getReferenceIdTable());
222 referenceTable
= referenceTable
.isEmpty() ?
" Reference" : referenceTable
+ " as Reference ";
223 String strIdFrom
= String
.format(strFrom
, referenceTable
);
225 String referenceFilter
= CdmUtils
.Nz(state
.getConfig().getReferenceIdTable());
226 if (! referenceFilter
.isEmpty()){
227 referenceFilter
= " AND " + referenceFilter
+ " ";
229 referenceFilter
= ""; //don't use it for now, in E+M the tabelle is directly used
231 String strIdQueryFirstPath
= strSelectId
+ strIdFrom
+ strOrderBy
;
232 String strIdQuerySecondPath
= strSelectId
+ strIdFrom
+ " AND (Reference.InRefFk is NOT NULL) ";
234 // if (config.getDoReferences() == CONCEPT_REFERENCES){
235 // strIdQueryNoInRef += " AND ( Reference.refId IN ( SELECT ptRefFk FROM PTaxon) ) " + referenceFilter;
238 String strRecordQuery
= strSelectFull
+ String
.format(strFrom
, " Reference ") + strWherePartitioned
+ strOrderBy
;
240 int recordsPerTransaction
= config
.getRecordsPerTransaction();
243 ResultSetPartitioner
<BerlinModelImportState
> partitioner
=
244 ResultSetPartitioner
.NewInstance(source
, strIdQueryFirstPath
, strRecordQuery
, recordsPerTransaction
);
245 while (partitioner
.nextPartition()){
246 partitioner
.doPartition(this, state
);
248 logger
.info("end make references without in-references ... " + getSuccessString(success
));
249 state
.setReferenceSecondPath(true);
252 // partitioner = ResultSetPartitioner.NewInstance(source, strIdQuerySecondPath, strRecordQuery, recordsPerTransaction);
253 // while (partitioner.nextPartition()){
254 // //currently not used as inRef assignment fully works through sorting of idQuery now, at least in E+M
255 // partitioner.doPartition(this, state);
257 // logger.info("end make references with no 1 in-reference ... " + getSuccessString(success));
258 state
.setReferenceSecondPath(false);
259 logger
.warn("Parsed book volumes: " + parsedBookVolumes
);
260 } catch (SQLException e
) {
261 logger
.error("SQLException:" + e
);
262 state
.setUnsuccessfull();
265 logger
.info("end make " + getPluralString() + " ... " + getSuccessString(success
));
267 state
.setUnsuccessfull();
269 deduplicationHelper
= null;
275 public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner
, BerlinModelImportState state
) {
276 deduplicationHelper
.restartSession();
278 if (state
.isReferenceSecondPath()){
279 return doPartitionSecondPath(partitioner
, state
);
281 boolean success
= true;
283 Map
<Integer
, Reference
> refToSave
= new HashMap
<>();
285 BerlinModelImportConfigurator config
= state
.getConfig();
290 RefCounter refCounter
= new RefCounter();
291 ResultSet rs
= partitioner
.getResultSet();
295 if ((i
++ % modCount
) == 0 && i
!= 1 ){ logger
.info("References handled: " + (i
-1) + " in round -" );}
297 success
&= makeSingleReferenceRecord(rs
, state
, partitioner
, refToSave
, refCounter
);
300 //for the concept reference a fixed uuid may be needed -> change uuid
301 Integer sourceSecId
= (Integer
)config
.getSourceSecId();
302 Reference sec
= refToSave
.get(sourceSecId
);
305 sec
.setUuid(config
.getSecUuid());
306 logger
.info("SecUuid changed to: " + config
.getSecUuid());
309 //save and store in map
310 logger
.warn("Save references (" + refCounter
.toString() + ")"); //set preliminary to warn for printing dedup count
312 getReferenceService().saveOrUpdate(refToSave
.values());
314 // logger.info("end makeReferences ..." + getSuccessString(success));;
316 } catch (SQLException e
) {
317 logger
.error("SQLException:" + e
);
325 * Adds the inReference to the according references.
330 private boolean doPartitionSecondPath(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner
, BerlinModelImportState state
) {
331 boolean success
= true;
333 Map
<Integer
, Reference
> refToSave
= new HashMap
<>();
335 @SuppressWarnings("unchecked")
336 Map
<String
, Reference
> relatedReferencesMap
= partitioner
.getObjectMap(REFERENCE_NAMESPACE
);
340 RefCounter refCounter
= new RefCounter();
342 ResultSet rs
= partitioner
.getResultSet();
345 if ((i
++ % modCount
) == 0 && i
!= 1 ){ logger
.info("References handled: " + (i
-1) + " in round -" );}
347 Integer refId
= rs
.getInt("refId");
348 Integer inRefFk
= nullSafeInt(rs
, "inRefFk");
350 if (inRefFk
!= null){
352 Reference thisRef
= relatedReferencesMap
.get(String
.valueOf(refId
));
354 Reference inRef
= relatedReferencesMap
.get(String
.valueOf(inRefFk
));
356 if (thisRef
!= null){
358 logger
.warn("No InRef found for nomRef: " + thisRef
.getTitleCache() + "; RefId: " + refId
+ "; inRefFK: " + inRefFk
);
360 thisRef
.setInReference(inRef
);
361 refToSave
.put(refId
, thisRef
);
362 if(!thisRef
.isProtectedTitleCache()){
363 thisRef
.setTitleCache(null);
364 thisRef
.getTitleCache();
367 logger
.warn("Reference which has an inReference not found in DB. RefId: " + refId
);
369 if(inRefFk
.equals(0)){
370 logger
.warn("InRefFk is 0 for refId "+ refId
);
376 //save and store in map
377 logger
.info("Save in references (" + refCounter
.toString() + ")");
378 getReferenceService().saveOrUpdate(refToSave
.values());
380 // }//end resultSetList
382 // logger.info("end makeReferences ..." + getSuccessString(success));;
384 } catch (SQLException e
) {
385 logger
.error("SQLException:" + e
);
392 public Map
<Object
, Map
<String
, ?
extends CdmBase
>> getRelatedObjectsForPartition(ResultSet rs
, BerlinModelImportState state
) {
397 Map
<Object
, Map
<String
, ?
extends CdmBase
>> result
= new HashMap
<>();
400 Set
<String
> teamIdSet
= new HashSet
<>();
401 Set
<String
> referenceIdSet
= new HashSet
<>();
402 Set
<String
> teamStringSet
= new HashSet
<>();
405 handleForeignKey(rs
, teamIdSet
, "NomAuthorTeamFk");
406 handleForeignKey(rs
, referenceIdSet
, "InRefFk");
407 handleForeignKey(rs
, teamStringSet
, "refAuthorString");
408 //TODO only needed in second path but state not available here to check if state is second path
409 handleForeignKey(rs
, referenceIdSet
, "refId");
412 Set
<String
> teamStringSet2
= new HashSet
<>();
413 for (String teamString
: teamStringSet
){
414 teamStringSet2
.add(teamString
.replace("'", "´"));
418 nameSpace
= BerlinModelAuthorTeamImport
.NAMESPACE
;
420 @SuppressWarnings("rawtypes")
421 Map
<String
, TeamOrPersonBase
> teamMap
= getCommonService().getSourcedObjectsByIdInSourceC(TeamOrPersonBase
.class, idSet
, nameSpace
);
422 result
.put(nameSpace
, teamMap
);
425 nameSpace
= REF_AUTHOR_NAMESPACE
;
426 idSet
= teamStringSet2
;
427 @SuppressWarnings("unchecked")
428 Map
<String
, TeamOrPersonBase
> refAuthorMap
= getCommonService().getSourcedObjectsByIdInSourceC(TeamOrPersonBase
.class, idSet
, nameSpace
);
429 result
.put(nameSpace
, refAuthorMap
);
432 nameSpace
= BerlinModelReferenceImport
.REFERENCE_NAMESPACE
;
433 idSet
= referenceIdSet
;
434 Map
<String
, Reference
> referenceMap
= getCommonService().getSourcedObjectsByIdInSourceC(Reference
.class, idSet
, nameSpace
);
435 result
.put(nameSpace
, referenceMap
);
437 } catch (SQLException e
) {
438 throw new RuntimeException(e
);
445 * Handles a single reference record
448 * @param biblioRefToSave
449 * @param nomRefToSave
450 * @param relatedBiblioReferences
451 * @param relatedNomReferences
455 private boolean makeSingleReferenceRecord(
457 BerlinModelImportState state
,
458 ResultSetPartitioner
<BerlinModelImportState
> partitioner
,
459 Map
<Integer
, Reference
> refToSave
,
460 RefCounter refCounter
){
462 boolean success
= true;
464 Integer refId
= null;
466 Map
<String
, Object
> valueMap
= getValueMap(rs
);
468 Integer categoryFk
= (Integer
)valueMap
.get("refCategoryFk".toLowerCase());
469 refId
= (Integer
)valueMap
.get("refId".toLowerCase());
470 Boolean thesisFlag
= (Boolean
)valueMap
.get("thesisFlag".toLowerCase());
474 logger
.debug("RefCategoryFk: " + categoryFk
);
477 reference
= makeThesis(valueMap
);
478 }else if (categoryFk
== REF_JOURNAL
){
479 reference
= makeJournal(valueMap
);
480 }else if(categoryFk
== REF_BOOK
){
481 reference
= makeBook(valueMap
);
482 }else if(categoryFk
== REF_DATABASE
){
483 reference
= makeDatabase(valueMap
);
484 }else if(categoryFk
== REF_INFORMAL
){
485 reference
= makeInformal(valueMap
);
486 }else if(categoryFk
== REF_WEBSITE
){
487 reference
= makeWebSite(valueMap
);
488 }else if(categoryFk
== REF_UNKNOWN
){
489 reference
= makeUnknown(valueMap
);
490 }else if(categoryFk
== REF_PRINT_SERIES
){
491 reference
= makePrintSeries(valueMap
);
492 }else if(categoryFk
== REF_CONFERENCE_PROCEEDINGS
){
493 reference
= makeProceedings(valueMap
);
494 }else if(categoryFk
== REF_ARTICLE
){
495 reference
= makeArticle(valueMap
);
496 }else if(categoryFk
== REF_JOURNAL_VOLUME
){
497 reference
= makeJournalVolume(valueMap
);
498 }else if(categoryFk
== REF_PART_OF_OTHER_TITLE
){
499 reference
= makePartOfOtherTitle(valueMap
);
501 logger
.warn("Unknown categoryFk (" + categoryFk
+ "). Create 'Generic instead'");
502 reference
= ReferenceFactory
.newGeneric();
507 String refYear
= (String
)valueMap
.get("refYear".toLowerCase());
508 reference
.setDatePublished(ImportHelper
.getDatePublished(refYear
));
510 handleEdition(reference
);
512 //created, updated, notes
513 doCreatedUpdatedNotes(state
, reference
, rs
);
515 //idInSource (import from older source to berlin model)
516 //TODO do we want this being imported? Maybe as alternatvie identifier?
517 String idInSource
= (String
)valueMap
.get("IdInSource".toLowerCase());
518 if (isNotBlank(idInSource
)){
519 if(!state
.getConfig().isDoSourceNumber()){
520 IdentifiableSource source
= IdentifiableSource
.NewDataImportInstance(idInSource
);
521 source
.setIdNamespace("import to Berlin Model");
522 reference
.addSource(source
);
524 makeSourceNumbers(state
, idInSource
, reference
, refId
);
528 if (resultSetHasColumn(rs
,"UUID")){
529 uuid
= rs
.getString("UUID");
531 reference
.setUuid(UUID
.fromString(uuid
));
535 //nom&BiblioReference - must be last because a clone is created
536 success
&= makeNomAndBiblioReference(rs
, state
, partitioner
, refId
, reference
, refCounter
, refToSave
);
539 } catch (Exception e
) {
540 logger
.warn("Reference with BM refId '" + CdmUtils
.Nz(refId
) + "' threw Exception and could not be saved");
554 private void makeSourceNumbers(BerlinModelImportState state
, String idInSource
, Reference reference
,
556 String
[] splits
= idInSource
.split("\\|");
557 for (String split
: splits
){
558 split
= split
.trim();
559 UUID uuid
= BerlinModelTransformer
.uuidEMReferenceSourceNumber
;
560 TermVocabulary
<DefinedTerm
> voc
= null; //user defined voc
561 DefinedTerm type
= getIdentiferType(state
, uuid
, "E+M Reference Source Number", "Euro+Med Reference Source Number", "E+M Source Number", voc
);
562 Identifier
.NewInstance(reference
, split
, type
);
569 private void handleEdition(Reference reference
) {
570 if (reference
.getEdition()!= null && reference
.getEdition().startsWith("ed. ")){
571 reference
.setEdition(reference
.getEdition().substring(4));
577 * Creates and saves a nom. reference and a biblio. reference after checking necessity
582 * @param biblioRefToSave
583 * @param nomRefToSave
587 * @throws SQLException
589 private boolean makeNomAndBiblioReference(
591 BerlinModelImportState state
,
592 @SuppressWarnings("rawtypes") ResultSetPartitioner partitioner
,
595 RefCounter refCounter
,
596 Map
<Integer
, Reference
> refToSave
597 ) throws SQLException
{
599 @SuppressWarnings("unchecked")
600 Map
<String
, Team
> teamMap
= partitioner
.getObjectMap(BerlinModelAuthorTeamImport
.NAMESPACE
);
602 String refCache
= trim(rs
.getString("refCache"));
603 String nomRefCache
= trim(rs
.getString("nomRefCache"));
604 String title
= trim(rs
.getString("title"));
605 String nomTitleAbbrev
= trim(rs
.getString("nomTitleAbbrev"));
606 boolean isPreliminary
= rs
.getBoolean("PreliminaryFlag");
607 String refAuthorString
= trim(rs
.getString("refAuthorString"));
608 Integer nomAuthorTeamFk
= nullSafeInt(rs
, "NomAuthorTeamFk");
609 Integer inRefFk
= nullSafeInt(rs
, "inRefFk");
612 TeamOrPersonBase
<?
> nomAuthor
= null;
613 if (nomAuthorTeamFk
!= null){
614 String strNomAuthorTeamFk
= String
.valueOf(nomAuthorTeamFk
);
615 nomAuthor
= teamMap
.get(strNomAuthorTeamFk
);
616 if (nomAuthor
== null){
617 logger
.warn("NomAuthor ("+strNomAuthorTeamFk
+") not found in teamMap (but it should exist) for " + refId
);
621 Reference sourceReference
= state
.getTransactionalSourceReference();
625 ref
.setAbbrevTitleCache(nomRefCache
, true);
626 ref
.setTitleCache(refCache
, true);
630 if (isNotBlank(nomTitleAbbrev
)){
631 ref
.setAbbrevTitle(nomTitleAbbrev
);
633 if (isNotBlank(title
)){
638 TeamOrPersonBase
<?
> author
= getAuthorship(state
, refAuthorString
, nomAuthor
, refId
);
639 ref
.setAuthorship(author
);
641 if (ref
.getType().equals(ReferenceType
.Book
)){
642 extraktBookVolume(ref
);
646 Reference inRef
= null;
647 if (inRefFk
!= null){
648 @SuppressWarnings({"unchecked" })
649 Map
<String
, Reference
> relatedReferences
= partitioner
.getObjectMap(REFERENCE_NAMESPACE
);
650 inRef
= relatedReferences
.get(String
.valueOf(inRefFk
));
652 inRef
= refToSave
.get(inRefFk
);
655 logger
.warn("InRef not (yet) found. RefId: " + refId
+ "; InRef: "+ inRefFk
);
657 ref
.setInReference(inRef
);
661 Reference result
= deduplicateReference(state
, ref
);
663 //dedup not possible at this point because inRef exists but is not yet defined
664 if (inRefFk
!= null && inRef
== null){
666 logger
.warn("Ref has deduplication candidate but inRef is still missing. " + inRef
);
668 logger
.debug("Reference was deduplicated. RefId: " + refId
);
669 //FIXME also check annotations etc. for deduplication
670 refCounter
.dedupCount
++;
673 refCounter
.refCount
++;
677 if (! refToSave
.containsKey(refId
)){
678 refToSave
.put(refId
, result
);
681 logger
.warn("Duplicate refId in Berlin Model database. Second reference was not imported !!");
686 ImportHelper
.setOriginalSource(result
, sourceReference
, refId
, REFERENCE_NAMESPACE
);
688 if (commonNameRefSet
!= null && commonNameRefSet
.contains(refId
)){
689 result
.addMarker(Marker
.NewInstance(MarkerType
.COMMON_NAME_REFERENCE(), true));
699 private String
trim(String string
) {
703 return string
.trim();
708 * Copies the created and updated information from the nomReference to the cloned bibliographic reference
709 * @param referenceBase
710 * @param nomReference
712 private void copyCreatedUpdated(Reference biblioReference
, Reference nomReference
) {
713 biblioReference
.setCreatedBy(nomReference
.getCreatedBy());
714 biblioReference
.setCreated(nomReference
.getCreated());
715 biblioReference
.setUpdatedBy(nomReference
.getUpdatedBy());
716 biblioReference
.setUpdated(nomReference
.getUpdated());
720 private Reference
makeArticle (Map
<String
, Object
> valueMap
){
722 IArticle article
= ReferenceFactory
.newArticle();
723 Object inRefFk
= valueMap
.get("inRefFk".toLowerCase());
724 Integer inRefCategoryFk
= (Integer
)valueMap
.get("inRefCategoryFk".toLowerCase());
725 Integer refId
= (Integer
)valueMap
.get("refId".toLowerCase());
727 if (inRefFk
!= null){
728 if (inRefCategoryFk
!= REF_JOURNAL
){
729 logger
.warn("Wrong inrefCategory for Article (refID = " + refId
+"). Type must be 'Journal' but was not (RefCategoryFk=" + inRefCategoryFk
+ "))." +
730 " InReference was added anyway! ");
733 logger
.warn ("Article has no inreference: " + refId
);
735 makeStandardMapper(valueMap
, (Reference
)article
); //url, pages, series, volume
736 String url
= (String
)valueMap
.get("url");
737 if (url
!= null && url
.contains("dx.doi.org")){
738 article
.setDoi(DOI
.fromString(url
));
739 article
.setUri(null);
741 return (Reference
)article
;
744 private Reference
makePartOfOtherTitle (Map
<String
, Object
> valueMap
){
747 Object inRefFk
= valueMap
.get("inRefFk".toLowerCase());
748 Integer inRefCategoryFk
= (Integer
)valueMap
.get("inRefCategoryFk".toLowerCase());
749 Integer refId
= (Integer
)valueMap
.get("refId".toLowerCase());
751 if (inRefCategoryFk
== null){
753 logger
.warn("Part-Of-Other-Title has no inRefCategoryFk! RefId = " + refId
+ ". ReferenceType set to Generic.");
754 result
= makeUnknown(valueMap
);
755 }else if (inRefFk
== null){
756 //TODO is this correct ??
757 logger
.warn("Part-Of-Other-Title has no in reference: " + refId
);
758 result
= makeUnknown(valueMap
);
759 }else if (inRefCategoryFk
== REF_BOOK
){
761 IBookSection bookSection
= ReferenceFactory
.newBookSection();
762 result
= (Reference
)bookSection
;
763 }else if (inRefCategoryFk
== REF_ARTICLE
){
765 logger
.info("Reference (refId = " + refId
+ ") of type 'part_of_other_title' is part of 'article'." +
766 " We use the section reference type for such in references now.") ;
767 result
= ReferenceFactory
.newSection();
768 }else if (inRefCategoryFk
== REF_JOURNAL
){
770 logger
.warn("Reference (refId = " + refId
+ ") of type 'part_of_other_title' has inReference of type 'journal'." +
771 " This is not allowed! Generic reference created instead") ;
772 result
= ReferenceFactory
.newGeneric();
773 result
.addMarker(Marker
.NewInstance(MarkerType
.TO_BE_CHECKED(), true));
774 }else if (inRefCategoryFk
== REF_PART_OF_OTHER_TITLE
){
775 logger
.info("Reference (refId = " + refId
+ ") of type 'part_of_other_title' has inReference 'part of other title'." +
776 " This is allowed, but may be true only for specific cases (e.g. parts of book chapters). You may want to check if this is correct") ;
777 result
= ReferenceFactory
.newSection();
779 logger
.warn("InReference type (catFk = " + inRefCategoryFk
+ ") of part-of-reference not recognized for refId " + refId
+ "." +
780 " Create 'Generic' reference instead");
781 result
= ReferenceFactory
.newGeneric();
783 makeStandardMapper(valueMap
, result
); //url, pages
790 * @param biblioRefToSave
791 * @param nomRefToSave
792 * @param relatedBiblioReferences
793 * @param relatedNomReferences
796 private boolean existsInMapOrToSave(Integer inRefFkInt
, Map
<Integer
, Reference
> refToSave
, Map
<String
, Reference
> relatedReferences
) {
797 boolean result
= false;
798 if (inRefFkInt
== null){
801 result
|= refToSave
.containsKey(inRefFkInt
);
802 result
|= relatedReferences
.containsKey(String
.valueOf(inRefFkInt
));
806 private Reference
makeWebSite(Map
<String
, Object
> valueMap
){
807 if (logger
.isDebugEnabled()){logger
.debug("RefType 'Website'");}
808 Reference webPage
= ReferenceFactory
.newWebPage();
809 makeStandardMapper(valueMap
, webPage
); //placePublished, publisher
813 private Reference
makeUnknown(Map
<String
, Object
> valueMap
){
814 if (logger
.isDebugEnabled()){logger
.debug("RefType 'Unknown'");}
815 Reference generic
= ReferenceFactory
.newGeneric();
816 // generic.setSeries(series);
817 makeStandardMapper(valueMap
, generic
); //pages, placePublished, publisher, series, volume
821 private Reference
makeInformal(Map
<String
, Object
> valueMap
){
822 if (logger
.isDebugEnabled()){logger
.debug("RefType 'Informal'");}
823 Reference generic
= ReferenceFactory
.newGeneric();
824 // informal.setSeries(series);
825 makeStandardMapper(valueMap
, generic
);//editor, pages, placePublished, publisher, series, volume
826 String informal
= (String
)valueMap
.get("InformalRefCategory".toLowerCase());
827 if (isNotBlank(informal
) ){
828 generic
.addExtension(informal
, ExtensionType
.INFORMAL_CATEGORY());
833 private Reference
makeDatabase(Map
<String
, Object
> valueMap
){
834 if (logger
.isDebugEnabled()){logger
.debug("RefType 'Database'");}
835 Reference database
= ReferenceFactory
.newDatabase();
836 makeStandardMapper(valueMap
, database
); //?
840 private Reference
makeJournal(Map
<String
, Object
> valueMap
){
841 if (logger
.isDebugEnabled()){logger
.debug("RefType 'Journal'");}
842 Reference journal
= ReferenceFactory
.newJournal();
844 Set
<String
> omitAttributes
= new HashSet
<>();
845 String series
= "series";
846 // omitAttributes.add(series);
848 makeStandardMapper(valueMap
, journal
, omitAttributes
); //issn,placePublished,publisher
849 // if (valueMap.get(series) != null){
850 // logger.warn("Series not yet implemented for journal!");
855 private Reference
makeBook(
856 Map
<String
, Object
> valueMap
){
858 if (logger
.isDebugEnabled()){logger
.debug("RefType 'Book'");}
859 Reference book
= ReferenceFactory
.newBook();
860 // Integer refId = (Integer)valueMap.get("refId".toLowerCase());
862 //Set bookAttributes = new String[]{"edition", "isbn", "pages","publicationTown","publisher","volume"};
864 Set
<String
> omitAttributes
= new HashSet
<>();
865 String attrSeries
= "series";
866 // omitAttributes.add(attrSeries);
868 makeStandardMapper(valueMap
, book
, omitAttributes
);
871 IPrintSeries printSeries
= null;
872 if (valueMap
.get(attrSeries
) != null){
873 String series
= (String
)valueMap
.get("title".toLowerCase());
875 String nomTitle
= (String
)valueMap
.get("nomTitleAbbrev".toLowerCase());
878 printSeries
= ReferenceFactory
.newPrintSeries(series
);
879 logger
.info("Implementation of printSeries is preliminary");
881 //Series (as Reference)
882 if (book
.getInSeries() != null && printSeries
!= null){
883 logger
.warn("Book has series string and inSeries reference. Can not take both. Series string neglected");
885 book
.setInSeries(printSeries
);
887 book
.setEditor(null);
894 int parsedBookVolumes
= 0;
895 private void extraktBookVolume(Reference book
) {
896 if (isExtractBookVolumeCandidate(book
)){
897 String patternStr
= "(.{2,})\\s(\\d{1,2})";
899 Pattern pattern
= Pattern
.compile(patternStr
);
901 String abbrevCache
= book
.getAbbrevTitleCache();
902 String titleCache
= book
.getTitleCache();
904 String volFull
= null;
905 String abbrev
= book
.getAbbrevTitle();
906 if (isNotBlank(abbrev
)){
907 Matcher matcher
= pattern
.matcher(abbrev
);
908 if (matcher
.matches()){
909 vol
= matcher
.group(groupIndex
);
910 abbrev
= matcher
.group(1);
914 String full
= book
.getTitle();
915 if (isNotBlank(full
)){
916 Matcher matcher
= pattern
.matcher(full
);
917 if (matcher
.matches()){
918 volFull
= matcher
.group(groupIndex
);
919 full
= matcher
.group(1);
922 if (vol
!= null && volFull
!= null){
923 if (!vol
.equals(volFull
)){
926 }else if (vol
== null && volFull
== null){
928 }else if (vol
== null){
929 if (isNotBlank(abbrev
)){
934 }else if (volFull
== null){
935 if (isNotBlank(full
)){
939 logger
.warn("Should not happen");
942 book
.setAbbrevTitle(abbrev
);
944 if (!book
.getAbbrevTitleCache().equals(abbrevCache
)){
945 logger
.warn("Abbrev title cache for parsed book volume does not match: " + book
.getAbbrevTitleCache() + " <-> "+abbrevCache
);
946 }else if (!book
.getTitleCache().equals(titleCache
)){
947 logger
.warn("Title cache for parsed book volume does not match: " + book
.getTitleCache() + " <-> "+titleCache
);
949 // System.out.println(titleCache);
950 // System.out.println(abbrevCache);
962 private boolean isExtractBookVolumeCandidate(Reference book
) {
963 if (isNotBlank(book
.getVolume()) || isNotBlank(book
.getEdition()) || isNotBlank(book
.getSeriesPart())){
966 if (!checkExtractBookVolumeTitle(book
.getAbbrevTitle())){
969 if (!checkExtractBookVolumeTitle(book
.getTitle())){
979 private boolean checkExtractBookVolumeTitle(String title
) {
983 if (title
.contains(",") || title
.contains("ed.") || title
.contains("Ed.")|| title
.contains("Suppl")
984 || title
.contains("Ser.")|| title
.contains("ser.")) {
991 * Returns the requested object if it exists in one of both maps. Prefers the refToSaveMap in ambigious cases.
993 * @param nomRefToSave
994 * @param relatedNomReferences
997 private Reference
getReferenceFromMaps(
999 Map
<Integer
, Reference
> refToSaveMap
,
1000 Map
<String
, Reference
> relatedRefMap
) {
1001 Reference result
= null;
1002 result
= refToSaveMap
.get(inRefFkInt
);
1003 if (result
== null){
1004 result
= relatedRefMap
.get(String
.valueOf(inRefFkInt
));
1009 private Reference
makePrintSeries(Map
<String
, Object
> valueMap
){
1010 if (logger
.isDebugEnabled()){logger
.debug("RefType 'PrintSeries'");}
1011 Reference printSeries
= ReferenceFactory
.newPrintSeries();
1012 makeStandardMapper(valueMap
, printSeries
, null);
1016 private Reference
makeProceedings(Map
<String
, Object
> valueMap
){
1017 if (logger
.isDebugEnabled()){logger
.debug("RefType 'Proceedings'");}
1018 Reference proceedings
= ReferenceFactory
.newProceedings();
1019 makeStandardMapper(valueMap
, proceedings
, null);
1023 private Reference
makeThesis(Map
<String
, Object
> valueMap
){
1024 if (logger
.isDebugEnabled()){logger
.debug("RefType 'Thesis'");}
1025 Reference thesis
= ReferenceFactory
.newThesis();
1026 makeStandardMapper(valueMap
, thesis
, null);
1031 private Reference
makeJournalVolume(Map
<String
, Object
> valueMap
){
1032 if (logger
.isDebugEnabled()){logger
.debug("RefType 'JournalVolume'");}
1033 //Proceedings proceedings = Proceedings.NewInstance();
1034 Reference journalVolume
= ReferenceFactory
.newGeneric();
1035 makeStandardMapper(valueMap
, journalVolume
, null);
1036 logger
.warn("Journal volumes not yet implemented. Generic created instead but with errors");
1037 return journalVolume
;
1040 private boolean makeStandardMapper(Map
<String
, Object
> valueMap
, Reference ref
){
1041 return makeStandardMapper(valueMap
, ref
, null);
1045 private boolean makeStandardMapper(Map
<String
, Object
> valueMap
, CdmBase cdmBase
, Set
<String
> omitAttributes
){
1046 boolean result
= true;
1047 for (CdmAttributeMapperBase mapper
: classMappers
){
1048 if (mapper
instanceof CdmSingleAttributeMapperBase
){
1049 result
&= makeStandardSingleMapper(valueMap
, cdmBase
, (CdmSingleAttributeMapperBase
)mapper
, omitAttributes
);
1050 }else if (mapper
instanceof CdmOneToManyMapper
){
1051 result
&= makeMultipleValueAddMapper(valueMap
, cdmBase
, (CdmOneToManyMapper
)mapper
, omitAttributes
);
1053 logger
.error("Unknown mapper type");
1060 private boolean makeStandardSingleMapper(Map
<String
, Object
> valueMap
, CdmBase cdmBase
, CdmSingleAttributeMapperBase mapper
, Set
<String
> omitAttributes
){
1061 boolean result
= true;
1062 if (omitAttributes
== null){
1063 omitAttributes
= new HashSet
<>();
1065 if (mapper
instanceof DbImportExtensionMapper
){
1066 result
&= ((DbImportExtensionMapper
)mapper
).invoke(valueMap
, cdmBase
);
1067 }else if (mapper
instanceof DbImportMarkerMapper
){
1068 result
&= ((DbImportMarkerMapper
)mapper
).invoke(valueMap
, cdmBase
);
1070 String sourceAttribute
= mapper
.getSourceAttributeList().get(0).toLowerCase();
1071 Object value
= valueMap
.get(sourceAttribute
);
1072 if (mapper
instanceof CdmUriMapper
&& value
!= null){
1074 value
= new URI (value
.toString());
1075 } catch (URISyntaxException e
) {
1076 logger
.error("URI syntax exception: " + value
.toString());
1081 String destinationAttribute
= mapper
.getDestinationAttribute();
1082 if (! omitAttributes
.contains(destinationAttribute
)){
1083 result
&= ImportHelper
.addValue(value
, cdmBase
, destinationAttribute
, mapper
.getTypeClass(), OVERWRITE
, OBLIGATORY
);
1091 private boolean makeMultipleValueAddMapper(Map
<String
, Object
> valueMap
, CdmBase cdmBase
, CdmOneToManyMapper
<CdmBase
, CdmBase
, CdmSingleAttributeMapperBase
> mapper
, Set
<String
> omitAttributes
){
1092 if (omitAttributes
== null){
1093 omitAttributes
= new HashSet
<>();
1095 boolean result
= true;
1096 String destinationAttribute
= mapper
.getSingleAttributeName();
1097 List
<Object
> sourceValues
= new ArrayList
<>();
1098 List
<Class
> classes
= new ArrayList
<>();
1099 for (CdmSingleAttributeMapperBase singleMapper
: mapper
.getSingleMappers()){
1100 String sourceAttribute
= singleMapper
.getSourceAttribute();
1101 Object value
= valueMap
.get(sourceAttribute
);
1102 sourceValues
.add(value
);
1103 Class
<?
> clazz
= singleMapper
.getTypeClass();
1107 result
&= ImportHelper
.addMultipleValues(sourceValues
, cdmBase
, destinationAttribute
, classes
, NO_OVERWRITE
, OBLIGATORY
);
1112 private TeamOrPersonBase
<?
> getAuthorship(BerlinModelImportState state
, String refAuthorString
,
1113 TeamOrPersonBase
<?
> nomAuthor
, Integer refId
){
1115 TeamOrPersonBase
<?
> result
;
1116 if (nomAuthor
!= null){
1118 if (isNotBlank(refAuthorString
) && !nomAuthor
.getTitleCache().equals(refAuthorString
)){
1119 boolean isSimilar
= handleSimilarAuthors(state
, refAuthorString
, nomAuthor
, refId
);
1121 String message
= "refAuthorString differs from nomAuthor.titleCache: " + refAuthorString
1122 + " <-> " + nomAuthor
.getTitleCache() + "; RefId: " + refId
;
1123 logger
.warn(message
);
1126 } else if (isNotBlank(refAuthorString
)){//only RefAuthorString exists
1127 refAuthorString
= refAuthorString
.trim();
1128 //TODO match with existing Persons/Teams
1129 TeamOrPersonBase
<?
> author
= state
.getRelatedObject(REF_AUTHOR_NAMESPACE
, refAuthorString
, TeamOrPersonBase
.class);
1130 if (author
== null){
1131 if (!BerlinModelAuthorTeamImport
.hasTeamSeparator(refAuthorString
)){
1132 author
= makePerson(refAuthorString
, false, refId
);
1134 author
= makeTeam(state
, refAuthorString
, refId
);
1136 state
.addRelatedObject(REF_AUTHOR_NAMESPACE
, refAuthorString
, author
);
1137 result
= deduplicatePersonOrTeam(state
, author
);
1139 if (result
!= author
){
1140 logger
.debug("RefAuthorString author deduplicated " + author
);
1142 if (!importSourceExists(author
, refAuthorString
, REF_AUTHOR_NAMESPACE
, state
.getTransactionalSourceReference() )){
1143 author
.addImportSource(refAuthorString
, REF_AUTHOR_NAMESPACE
, state
.getTransactionalSourceReference(), null);
1147 logger
.debug("RefAuthor loaded from map");
1160 * @param refAuthorString
1164 private TeamOrPersonBase
<?
> makeTeam(BerlinModelImportState state
, String refAuthorString
, Integer refId
) {
1165 Team team
= Team
.NewInstance();
1166 boolean hasDedupMember
= false;
1167 if (containsEdOrColon(refAuthorString
)){
1168 team
.setTitleCache(refAuthorString
, true);
1170 String
[] refAuthorTeams
= BerlinModelAuthorTeamImport
.splitTeam(refAuthorString
);
1171 boolean lastWasInitials
= false;
1172 for (int i
= 0; i
< refAuthorTeams
.length
;i
++){
1173 if (lastWasInitials
){
1174 lastWasInitials
= false;
1177 String fullTeam
= refAuthorTeams
[i
].trim();
1178 String initials
= null;
1179 if (refAuthorTeams
.length
> i
+1){
1180 String nextSplit
= refAuthorTeams
[i
+1].trim();
1181 if (isInitial(nextSplit
)){
1182 lastWasInitials
= true;
1183 initials
= nextSplit
;
1186 Person member
= makePerson(fullTeam
, isNotBlank(initials
), refId
);
1188 if (initials
!= null){
1189 if (member
.getInitials() != null){
1190 logger
.warn("Initials already set: " + refId
);
1191 }else if (!member
.isProtectedTitleCache()){
1192 member
.setInitials(initials
);
1194 member
.setTitleCache(member
.getTitleCache() + ", " + initials
, true);
1198 if (i
== refAuthorTeams
.length
-1 && BerlinModelAuthorTeamImport
.isEtAl(member
)){
1199 team
.setHasMoreMembers(true);
1201 Person dedupMember
= deduplicatePersonOrTeam(state
, member
);
1202 if (dedupMember
!= member
){
1203 hasDedupMember
= true;
1205 if (!importSourceExists(member
, refAuthorString
, REF_AUTHOR_NAMESPACE
, state
.getTransactionalSourceReference())){
1206 member
.addImportSource(refAuthorString
, REF_AUTHOR_NAMESPACE
, state
.getTransactionalSourceReference(), null);
1210 team
.addTeamMember(dedupMember
);
1215 TeamOrPersonBase
<?
> result
= team
;
1216 if (team
.getTeamMembers().size() == 1 && !team
.isHasMoreMembers()){
1217 Person person
= team
.getTeamMembers().get(0);
1218 checkPerson(person
, refAuthorString
, hasDedupMember
, refId
);
1221 checkTeam(team
, refAuthorString
, refId
);
1228 private static void checkTeam(Team team
, String refAuthorString
, Integer refId
) {
1229 TeamDefaultCacheStrategy formatter
= (TeamDefaultCacheStrategy
) team
.getCacheStrategy();
1231 if (formatter
.getTitleCache(team
).equals(refAuthorString
)){
1232 team
.setProtectedTitleCache(false);
1233 }else if(formatter
.getTitleCache(team
).replace(" & ", ", ").equals(refAuthorString
.replace(" & ", ", ").replace(" ,", ","))){
1234 //also accept teams with ', ' as final member separator as not protected
1235 team
.setProtectedTitleCache(false);
1236 }else if(formatter
.getFullTitle(team
).replace(" & ", ", ").equals(refAuthorString
.replace(" & ", ", "))){
1237 //.. or teams with initials first
1238 team
.setProtectedTitleCache(false);
1239 }else if (containsEdOrColon(refAuthorString
)){
1240 //nothing to do, it is expected to be protected
1243 team
.setTitleCache(refAuthorString
, true);
1244 logger
.warn("Creation of titleCache for team with members did not (fully) work: " + refAuthorString
+ " <-> " + formatter
.getTitleCache(team
)+ " : " + refId
);
1248 private static void checkPerson(Person person
, String refAuthorString
, boolean hasDedupMember
, Integer refId
) {
1249 PersonDefaultCacheStrategy formatter
= (PersonDefaultCacheStrategy
) person
.getCacheStrategy();
1251 String oldTitleCache
= person
.getTitleCache();
1252 boolean oldTitleCacheProtected
= person
.isProtectedTitleCache();
1254 if (! oldTitleCache
.equals(refAuthorString
)){
1255 logger
.error("Old titleCache does not equal refAuthorString this should not happen. "+ oldTitleCache
+ " <-> " + refAuthorString
+ "; refId = " + refId
);
1258 boolean protect
= true;
1259 person
.setProtectedTitleCache(false);
1260 if (refAuthorString
.equals(formatter
.getTitleCache(person
))){
1262 }else if(formatter
.getFullTitle(person
).equals(refAuthorString
)){
1263 //.. or teams with initials first
1266 //keep protected, see below
1269 if (hasDedupMember
){
1271 //TODO maybe even do not use dedup for testing
1272 person
.setTitleCache(oldTitleCache
, oldTitleCacheProtected
);
1273 if (protect
!= oldTitleCacheProtected
){
1274 logger
.warn("Deduplicated person protection requirement unclear for "+refAuthorString
+". New:"+protect
+"/Old:"+oldTitleCacheProtected
+"; RefId: " + refId
);
1278 logger
.warn("Creation of titleCache for person (converted from team) with members did not (fully) work: " + refAuthorString
+ " <-> " + formatter
.getTitleCache(person
)+ " : " + refId
);
1279 person
.setTitleCache(refAuthorString
, protect
);
1286 private static boolean containsEdOrColon(String str
) {
1287 if (str
.contains(" ed.") || str
.contains(" Ed.") || str
.contains("(ed.")
1288 || str
.contains("[ed.") || str
.contains("(Eds)") || str
.contains("(Eds.)") ||
1289 str
.contains("(eds.)") || str
.contains(":")|| str
.contains(";") || str
.contains("Publ. & Inform. Directorate")
1290 || str
.contains("Anonymous [Department of Botany, Faculty of Science, FER-ZPR, University of Zagreb]")
1291 || str
.contains("Davis, P. H. (Güner, A. & al.)")){
1302 private static boolean isInitial(String str
) {
1306 boolean matches
= str
.trim().matches("(\\p{javaUpperCase}|Yu|Ya|Th|Ch|Lj|Sz|Dz|Sh|Ju|R. M. da S)\\.?"
1307 + "(\\s*[-\\s]\\s*(\\p{javaUpperCase}|Yu|Ja|Kh|Tz|Ya|Th|Ju)\\.?)*(\\s+(van|von|de|de la|del|da|van der))?");
1311 private <T
extends TeamOrPersonBase
<?
>> T
deduplicatePersonOrTeam(BerlinModelImportState state
,T author
) {
1312 T result
= deduplicationHelper
.getExistingAuthor(state
, author
);
1316 private Reference
deduplicateReference(BerlinModelImportState state
,Reference ref
) {
1317 Reference result
= deduplicationHelper
.getExistingReference(state
, ref
);
1321 private static Person
makePerson(String full
, boolean followedByInitial
, Integer refId
) {
1322 Person person
= Person
.NewInstance();
1323 person
.setTitleCache(full
, true);
1324 if (!full
.matches(".*[\\s\\.].*")){
1325 person
.setFamilyName(full
);
1326 person
.setProtectedTitleCache(false);
1328 parsePerson(person
, full
, true, followedByInitial
);
1331 if ((full
.length() <= 2 && !full
.matches("(Li|Bo|Em|Ay|Ma)")) || (full
.length() == 3 && full
.endsWith(".") && !full
.equals("al.")) ){
1332 logger
.warn("Unexpected short nom author name part: " + full
+ "; " + refId
);
1338 private static void parsePerson(Person person
, String str
, boolean preliminary
, boolean followedByInitial
) {
1339 String capWord
= "\\p{javaUpperCase}\\p{javaLowerCase}{2,}";
1340 String famStart
= "(Le |D'|'t |Mc|Mac|Des |d'|Du |De |Al-)";
1341 String regEx
= "((\\p{javaUpperCase}|Ya|Th|Ju|Kh|An)\\.([\\s-]\\p{javaUpperCase}\\.)*(\\s(de|del|da|von|van|van der|v.|af|zu|von M. Und L.))?\\s)("
1342 + famStart
+ "?" + capWord
+ "((-| y | i | é | de | de la )" + capWord
+ ")?)";
1343 Matcher matcher
= Pattern
.compile(regEx
).matcher(str
);
1344 if (matcher
.matches()){
1345 person
.setProtectedTitleCache(false);
1346 String familyName
= matcher
.group(6).trim();
1347 person
.setFamilyName(familyName
);
1348 person
.setInitials(matcher
.group(1).trim());
1350 String regEx2
= "("+ capWord
+ "\\s" + capWord
+ "|Le Sueur|Beck von Mannagetta|Di Martino|Galán de Mera|Van Der Maesen|Farga i Arquimbau|Perez de Paz|Borzatti de Loewenstern|Lo Giudice|Perez de Paz)";
1351 Matcher matcher2
= Pattern
.compile(regEx2
).matcher(str
);
1352 if (followedByInitial
&& matcher2
.matches()){
1353 person
.setFamilyName(str
);
1354 person
.setProtectedTitleCache(false);
1356 person
.setTitleCache(str
, preliminary
);
1361 private static boolean handleSimilarAuthors(BerlinModelImportState state
, String refAuthorString
,
1362 TeamOrPersonBase
<?
> nomAuthor
, int refId
) {
1363 String nomTitle
= nomAuthor
.getTitleCache();
1365 if (refAuthorString
.equals(nomAuthor
.getNomenclaturalTitleCache())){
1369 if (refAuthorString
.replace(" & ", ", ").equals(nomTitle
.replace(" & ", ", "))){
1370 //nomTitle equal except for "&"
1373 String nomFullTitle
= nomAuthor
.getFullTitle();
1374 if (refAuthorString
.replace(" & ", ", ").equals(nomFullTitle
.replace(" & ", ", "))){
1378 if (nomAuthor
.isInstanceOf(Person
.class)){
1379 Person person
= CdmBase
.deproxy(nomAuthor
, Person
.class);
1381 //refAuthor has initials behind, nom Author in front // the other way round is handled in firstIsFullNameOfInitialName
1382 if (refAuthorString
.contains(",") && !nomTitle
.contains(",") ){
1383 String
[] splits
= refAuthorString
.split(",");
1384 if (splits
.length
== 2){
1385 String newMatch
= splits
[1].trim() + " " + splits
[0].trim();
1386 if (newMatch
.equals(nomTitle
)){
1387 if (isBlank(person
.getFamilyName())){
1388 person
.setFamilyName(splits
[0].trim());
1390 if (isBlank(person
.getInitials())){
1391 person
.setInitials(splits
[1].trim());
1398 if (refAuthorIsFamilyAuthorOfNomAuthor(state
, refAuthorString
, person
)){
1402 if (firstIsFullNameOfInitialName(state
, refAuthorString
, person
, refId
)){
1413 * @param refAuthorString
1417 private static boolean refAuthorIsFamilyAuthorOfNomAuthor(BerlinModelImportState state
, String refAuthorString
,
1419 if (refAuthorString
.equals(person
.getFamilyName())){
1428 * @param refAuthorString
1432 private static boolean firstIsFullNameOfInitialName(BerlinModelImportState state
, String fullName
,
1433 Person initialAuthor
, int refId
) {
1434 String initialName
= initialAuthor
.getTitleCache();
1436 String
[] fullSplits
= fullName
.split(",");
1437 String
[] initialSplits
= initialName
.split(",");
1439 if (fullSplits
.length
== 2 && initialSplits
.length
== 2){
1440 String
[] fullGivenName
= fullSplits
[1].trim().split(" ");
1441 String
[] initialsGivenName
= initialSplits
[1].trim().split(" ");
1442 boolean result
= compareFamilyAndInitials(fullSplits
[0], initialSplits
[0], fullGivenName
, initialsGivenName
);
1444 setGivenName(state
, fullSplits
[1], initialAuthor
, refId
);
1447 }else if (fullSplits
.length
== 1 && initialSplits
.length
== 2){
1448 String
[] fullSingleSplits
= fullName
.split(" ");
1449 String fullFamily
= fullSingleSplits
[fullSingleSplits
.length
-1];
1450 String
[] fullGivenName
= Arrays
.copyOfRange(fullSingleSplits
, 0, fullSingleSplits
.length
-1);
1451 String
[] initialsGivenName
= initialSplits
[1].trim().split(" ");
1452 boolean result
= compareFamilyAndInitials(fullFamily
, initialSplits
[0], fullGivenName
, initialsGivenName
);
1454 if(hasAtLeastOneFullName(fullGivenName
)){
1455 setGivenName(state
, CdmUtils
.concat(" ", fullGivenName
), initialAuthor
, refId
);
1459 }else if (fullSplits
.length
== 1 && initialAuthor
.getInitials() == null){
1460 //don't if this will be implemented, initialAuthors with only nomencl.Author set
1467 * @param fullGivenName
1470 private static boolean hasAtLeastOneFullName(String
[] fullGivenName
) {
1471 for (String singleName
: fullGivenName
){
1472 if (!singleName
.endsWith(".") && singleName
.length() > 2 && !singleName
.matches("(von|van)") ){
1479 private static void setGivenName(BerlinModelImportState state
, String givenName
, Person person
, int refId
) {
1480 givenName
= givenName
.trim();
1481 if(person
.getGivenName() == null || person
.getGivenName().equals(givenName
)){
1482 person
.setGivenName(givenName
);
1484 logger
.warn("RefAuthor given name and existing given name differ: " + givenName
+ " <-> " + person
.getGivenName() + "; RefId + " + refId
);
1488 protected static boolean compareFamilyAndInitials(String fullFamilyName
, String initialsFamilyName
,
1489 String
[] fullGivenName
, String
[] initialsGivenName
) {
1490 if (!fullFamilyName
.equals(initialsFamilyName
)){
1493 if (fullGivenName
.length
== initialsGivenName
.length
){
1494 for (int i
=0; i
< fullGivenName
.length
; i
++){
1495 if (fullGivenName
[i
].length() == 0 //comma ending not allowed
1496 || initialsGivenName
[i
].length() != 2 //only K. or similar allowed
1497 || fullGivenName
[i
].length() < initialsGivenName
[i
].length() //fullFirstName must be longer than abbrev Name
1498 || !initialsGivenName
[i
].endsWith(".") //initials must end with "."
1499 || !fullGivenName
[i
].startsWith(initialsGivenName
[i
].replace(".", ""))){ //start with same letter
1500 if (fullGivenName
[i
].matches("(von|van|de|zu)") && fullGivenName
[i
].equals(initialsGivenName
[i
])){
1513 public Set
<String
> getObligatoryAttributes(boolean lowerCase
, BerlinModelImportConfigurator config
){
1514 Set
<String
> result
= new HashSet
<>();
1515 Class
<ICdmImport
>[] ioClassList
= config
.getIoClassList();
1516 result
.addAll(Arrays
.asList(unclearMappers
));
1517 result
.addAll(Arrays
.asList(createdAndNotesAttributes
));
1518 result
.addAll(Arrays
.asList(operationalAttributes
));
1519 CdmIoMapping mapping
= new CdmIoMapping();
1520 for (CdmAttributeMapperBase mapper
: classMappers
){
1521 mapping
.addMapper(mapper
);
1523 result
.addAll(mapping
.getSourceAttributes());
1525 Set
<String
> lowerCaseResult
= new HashSet
<>();
1526 for (String str
: result
){
1527 if (str
!= null){lowerCaseResult
.add(str
.toLowerCase());}
1529 result
= lowerCaseResult
;
1535 protected boolean doCheck(BerlinModelImportState state
){
1536 BerlinModelReferenceImportValidator validator
= new BerlinModelReferenceImportValidator();
1537 return validator
.validate(state
, this);
1541 protected boolean isIgnore(BerlinModelImportState state
){
1542 return (state
.getConfig().getDoReferences() == IImportConfigurator
.DO_REFERENCES
.NONE
);