2 * Copyright (C) 2017 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
9 package eu
.etaxonomy
.cdm
.dataInserter
;
11 import java
.util
.ArrayList
;
12 import java
.util
.Arrays
;
13 import java
.util
.EnumSet
;
14 import java
.util
.HashMap
;
15 import java
.util
.List
;
18 import java
.util
.UUID
;
20 import org
.apache
.log4j
.Level
;
21 import org
.apache
.log4j
.Logger
;
22 import org
.hibernate
.Session
;
23 import org
.springframework
.context
.event
.ContextRefreshedEvent
;
24 import org
.springframework
.security
.authentication
.AuthenticationCredentialsNotFoundException
;
25 import org
.springframework
.security
.core
.GrantedAuthority
;
26 import org
.springframework
.transaction
.TransactionStatus
;
27 import org
.springframework
.transaction
.annotation
.Transactional
;
29 import eu
.etaxonomy
.cdm
.api
.application
.AbstractDataInserter
;
30 import eu
.etaxonomy
.cdm
.api
.application
.CdmRepository
;
31 import eu
.etaxonomy
.cdm
.api
.service
.pager
.Pager
;
32 import eu
.etaxonomy
.cdm
.model
.agent
.Institution
;
33 import eu
.etaxonomy
.cdm
.model
.name
.NomenclaturalStatus
;
34 import eu
.etaxonomy
.cdm
.model
.name
.NomenclaturalStatusType
;
35 import eu
.etaxonomy
.cdm
.model
.name
.Rank
;
36 import eu
.etaxonomy
.cdm
.model
.name
.TaxonName
;
37 import eu
.etaxonomy
.cdm
.model
.permission
.CRUD
;
38 import eu
.etaxonomy
.cdm
.model
.permission
.GrantedAuthorityImpl
;
39 import eu
.etaxonomy
.cdm
.model
.permission
.Group
;
40 import eu
.etaxonomy
.cdm
.model
.permission
.PermissionClass
;
41 import eu
.etaxonomy
.cdm
.model
.taxon
.Taxon
;
42 import eu
.etaxonomy
.cdm
.model
.taxon
.TaxonBase
;
43 import eu
.etaxonomy
.cdm
.model
.term
.DefinedTerm
;
44 import eu
.etaxonomy
.cdm
.model
.term
.DefinedTermBase
;
45 import eu
.etaxonomy
.cdm
.model
.term
.TermVocabulary
;
46 import eu
.etaxonomy
.cdm
.persistence
.dao
.hibernate
.taxonGraph
.AbstractHibernateTaxonGraphProcessor
;
47 import eu
.etaxonomy
.cdm
.persistence
.dao
.taxonGraph
.TaxonGraphException
;
48 import eu
.etaxonomy
.cdm
.persistence
.permission
.CdmAuthority
;
49 import eu
.etaxonomy
.cdm
.persistence
.permission
.Role
;
50 import eu
.etaxonomy
.cdm
.vaadin
.model
.registration
.KindOfUnitTerms
;
51 import eu
.etaxonomy
.cdm
.vaadin
.permission
.RolesAndPermissions
;
54 // * Can create missing registrations for names which have Extensions of the Type <code>IAPTRegdata.json</code>.
55 //* See https://dev.e-taxonomy.eu/redmine/issues/6621 for further details.
56 //* This feature can be activated by by supplying one of the following jvm command line arguments:
58 //* <li><code>-DregistrationCreate=iapt</code>: create all iapt Registrations if missing</li>
59 //* <li><code>-DregistrationWipeout=iapt</code>: remove all iapt Registrations</li>
60 //* <li><code>-DregistrationWipeout=all</code>: remove all Registrations</li>
62 //* The <code>-DregistrationWipeout</code> commands are executed before the <code>-DregistrationCreate</code> and will not change the name and type designations.
65 * This feature can be activated by by supplying one of the following jvm command line arguments:
67 * <li><code>-DtaxonGraphCreate=true</code>: create taxon graph relations for all names below genus level</li>
70 * @author a.kohlbecker
73 public class RegistrationRequiredDataInserter
extends AbstractDataInserter
{
75 // protected static final String PARAM_NAME_CREATE = "registrationCreate";
77 // protected static final String PARAM_NAME_WIPEOUT = "registrationWipeout";
79 protected static final String TAXON_GRAPH_CREATE
= "taxonGraphCreate";
81 protected static final UUID GROUP_SUBMITTER_UUID
= UUID
.fromString("c468c6a7-b96c-4206-849d-5a825f806d3e");
83 protected static final UUID GROUP_CURATOR_UUID
= UUID
.fromString("135210d3-3db7-4a81-ab36-240444637d45");
85 private static final EnumSet
<CRUD
> CREATE_READ
= EnumSet
.of(CRUD
.CREATE
, CRUD
.READ
);
86 private static final EnumSet
<CRUD
> CREATE_READ_UPDATE_DELETE
= EnumSet
.of(CRUD
.CREATE
, CRUD
.READ
, CRUD
.UPDATE
, CRUD
.DELETE
);
88 private static final Logger logger
= Logger
.getLogger(RegistrationRequiredDataInserter
.class);
90 // private ExtensionType extensionTypeIAPTRegData;
92 Map
<String
, Institution
> instituteMap
= new HashMap
<>();
94 public static boolean commandsExecuted
= false;
96 private CdmRepository repo
;
98 private boolean hasRun
= false;
100 public void setCdmRepository(CdmRepository repo
){
105 public void onApplicationEvent(ContextRefreshedEvent event
) {
111 runAsAuthentication(Role
.ROLE_ADMIN
);
113 insertRequiredData();
114 executeSuppliedCommands();
116 restoreAuthentication();
122 private void insertRequiredData() {
124 TransactionStatus txStatus
= repo
.startTransaction(false);
126 Role roleCuration
= RolesAndPermissions
.ROLE_CURATION
;
127 if(repo
.getGrantedAuthorityService().find(roleCuration
.getUuid()) == null){
128 repo
.getGrantedAuthorityService().saveOrUpdate(roleCuration
.asNewGrantedAuthority());
131 Group groupCurator
= repo
.getGroupService().load(GROUP_CURATOR_UUID
, Arrays
.asList("grantedAuthorities"));
132 if(groupCurator
== null){
133 groupCurator
= Group
.NewInstance();
134 groupCurator
.setUuid(GROUP_CURATOR_UUID
);
135 groupCurator
.setName("Curator");
137 assureGroupHas(groupCurator
, new CdmAuthority(PermissionClass
.REGISTRATION
, CREATE_READ_UPDATE_DELETE
).toString());
138 repo
.getGroupService().saveOrUpdate(groupCurator
);
140 Group groupSubmitter
= repo
.getGroupService().load(GROUP_SUBMITTER_UUID
, Arrays
.asList("grantedAuthorities"));
141 if(groupSubmitter
== null){
142 groupSubmitter
= Group
.NewInstance();
143 groupSubmitter
.setUuid(GROUP_SUBMITTER_UUID
);
144 groupSubmitter
.setName("Submitter");
146 assureGroupHas(groupSubmitter
, new CdmAuthority(PermissionClass
.TAXONNAME
, CREATE_READ
).toString());
147 assureGroupHas(groupSubmitter
, new CdmAuthority(PermissionClass
.TEAMORPERSONBASE
, CREATE_READ
).toString());
148 assureGroupHas(groupSubmitter
, new CdmAuthority(PermissionClass
.REGISTRATION
, CREATE_READ
).toString());
149 assureGroupHas(groupSubmitter
, new CdmAuthority(PermissionClass
.REFERENCE
, CREATE_READ
).toString());
150 assureGroupHas(groupSubmitter
, new CdmAuthority(PermissionClass
.SPECIMENOROBSERVATIONBASE
, CREATE_READ
).toString());
151 assureGroupHas(groupSubmitter
, new CdmAuthority(PermissionClass
.COLLECTION
, CREATE_READ
).toString());
152 repo
.getGroupService().saveOrUpdate(groupSubmitter
);
154 TermVocabulary
<DefinedTerm
> kindOfUnitVocabulary
= repo
.getVocabularyService().find(KindOfUnitTerms
.KIND_OF_UNIT_VOCABULARY().getUuid());
155 if(kindOfUnitVocabulary
== null){
156 kindOfUnitVocabulary
= repo
.getVocabularyService().save(KindOfUnitTerms
.KIND_OF_UNIT_VOCABULARY());
159 DefinedTermBase
<?
> kouSpecimen
= repo
.getTermService().find(KindOfUnitTerms
.SPECIMEN().getUuid());
160 DefinedTermBase
<?
> kouImage
= repo
.getTermService().find(KindOfUnitTerms
.PUBLISHED_IMAGE().getUuid());
161 DefinedTermBase
<?
> kouUnpublishedImage
= repo
.getTermService().find(KindOfUnitTerms
.UNPUBLISHED_IMAGE().getUuid());
162 DefinedTermBase
<?
> kouCulture
= repo
.getTermService().find(KindOfUnitTerms
.CULTURE_METABOLIC_INACTIVE().getUuid());
164 if(kouSpecimen
== null){
165 kouSpecimen
= repo
.getTermService().save(KindOfUnitTerms
.SPECIMEN());
167 if(kouImage
== null){
168 kouImage
= repo
.getTermService().save(KindOfUnitTerms
.PUBLISHED_IMAGE());
170 if(kouUnpublishedImage
== null){
171 kouUnpublishedImage
= repo
.getTermService().save(KindOfUnitTerms
.UNPUBLISHED_IMAGE());
173 if(kouCulture
== null){
174 kouCulture
= repo
.getTermService().save(KindOfUnitTerms
.CULTURE_METABOLIC_INACTIVE());
177 Set
<DefinedTerm
> termInVocab
= kindOfUnitVocabulary
.getTerms();
178 List
<DefinedTermBase
> kouTerms
= Arrays
.asList(kouCulture
, kouImage
, kouSpecimen
, kouUnpublishedImage
);
180 for(DefinedTermBase
<?
> t
: kouTerms
){
181 if(!termInVocab
.contains(t
)){
182 kindOfUnitVocabulary
.addTerm((DefinedTerm
)t
);
186 repo
.commitTransaction(txStatus
);
190 private void assureGroupHas(Group group
, String authorityString
){
191 boolean authorityExists
= false;
193 for(GrantedAuthority ga
: group
.getGrantedAuthorities()){
194 if((authorityExists
= ga
.getAuthority().equals(authorityString
)) == true){
198 if(!authorityExists
){
199 group
.addGrantedAuthority(findGrantedAuthority(authorityString
));
203 private GrantedAuthorityImpl
findGrantedAuthority(String authorityString
){
204 GrantedAuthorityImpl ga
= null;
206 ga
= repo
.getGrantedAuthorityService().findAuthorityString(authorityString
);
207 } catch (AuthenticationCredentialsNotFoundException e
){
211 ga
= GrantedAuthorityImpl
.NewInstance(authorityString
);
212 repo
.getGrantedAuthorityService().save(ga
);
220 private void executeSuppliedCommands() {
222 if(commandsExecuted
){
224 // a second run could take place during initialization of the web context
227 commandsExecuted
= true;
229 String taxonGraphCreate
= System
.getProperty(TAXON_GRAPH_CREATE
);
231 if(taxonGraphCreate
!= null){
232 AbstractHibernateTaxonGraphProcessor processor
= new AbstractHibernateTaxonGraphProcessor() {
235 public Session
getSession() {
236 return repo
.getSession();
239 logger
.setLevel(Level
.DEBUG
);
240 int chunksize
= 1000;
242 TransactionStatus tx
;
243 Pager
<Taxon
> taxonPage
;
244 List
<TaxonBase
> taxa
= new ArrayList
<>();
245 logger
.debug("======= fixing sec refrences =========");
247 tx
= repo
.startTransaction(false);
248 taxonPage
= repo
.getTaxonService().page(Taxon
.class, chunksize
, pageIndex
++, null, null);
249 if(taxonPage
.getRecords().size() == 0){
250 repo
.commitTransaction(tx
);
253 for(Taxon taxon
: taxonPage
.getRecords()){
254 taxon
.setSec(processor
.secReference());
255 repo
.getTaxonService().saveOrUpdate(taxon
);
257 repo
.commitTransaction(tx
);
260 logger
.debug("======= creating taxon graph =========");
262 Pager
<TaxonName
> page
;
264 tx
= repo
.startTransaction(false);
265 page
= repo
.getNameService().page(null, chunksize
, pageIndex
++, null, null);
266 if(page
.getRecords().size() == 0){
267 repo
.commitTransaction(tx
);
270 logger
.debug(TAXON_GRAPH_CREATE
+ ": chunk " + pageIndex
+ "/" + Math
.ceil(page
.getCount() / chunksize
));
271 taxa
= new ArrayList
<>();
273 for(TaxonName name
: page
.getRecords()){
274 if(name
.getRank() != null && name
.getRank().isLower(Rank
.GENUS())){
275 NomenclaturalStatusType illegitimType
= findILegitimateStatusType(name
);
276 if(illegitimType
== null){
279 logger
.debug("Processing name: " + name
.getTitleCache() + " [" + name
.getRank().getLabel() + "]");
280 taxon
= processor
.assureSingleTaxon(name
);
281 processor
.updateEdges(taxon
);
283 } catch (TaxonGraphException e
) {
284 logger
.error(e
.getMessage());
287 logger
.debug("Skipping illegitimate name: " + name
.getTitleCache() + " " + illegitimType
.getLabel() + " [" + name
.getRank().getLabel() + "]");
290 logger
.debug("Skipping name: " + name
.getTitleCache() + " [" + (name
.getRank() != null ? name
.getRank().getLabel() : "NULL") + "]");
293 repo
.getTaxonService().saveOrUpdate(taxa
);
294 repo
.commitTransaction(tx
);
298 // String wipeoutCmd = System.getProperty(PARAM_NAME_WIPEOUT);
299 // String createCmd = System.getProperty(PARAM_NAME_CREATE);
301 // // ============ DELETE
302 // if(wipeoutCmd != null && wipeoutCmd.matches("iapt|all")){
304 // boolean onlyIapt = wipeoutCmd.equals("iapt");
305 // Set<UUID> deleteCandidates = new HashSet<UUID>();
307 // TransactionStatus tx = repo.startTransaction(true);
308 // List<Registration> allRegs = repo.getRegistrationService().list(null, null, null, null, null);
309 // for(Registration reg : allRegs){
312 // @SuppressWarnings("unchecked")
313 // Set<String> extensions = reg.getName().getExtensions(getExtensionTypeIAPTRegData());
314 // if(reg.getUuid() != null){
315 // deleteCandidates.add(reg.getUuid());
317 // } catch(NullPointerException e){
321 // if(reg.getUuid() != null){
322 // deleteCandidates.add(reg.getUuid());
326 // repo.commitTransaction(tx);
327 // if(!deleteCandidates.isEmpty()){
329 // repo.getRegistrationService().delete(deleteCandidates);
330 // } catch (Exception e) {
331 // // MySQLIntegrityConstraintViolationException happens here every second run !!!
337 // // ============ CREATE
338 // int pageIndex = 0;
339 // if(createCmd != null && createCmd.equals("iapt")){
341 // DateTimeFormatter dateFormat1 = org.joda.time.format.DateTimeFormat.forPattern("dd.MM.yy").withPivotYear(1950);
342 // DateTimeFormatter dateFormat2 = org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd").withPivotYear(1950);
344 // TransactionStatus tx = repo.startTransaction(false);
346 // Pager<TaxonName> pager = repo.getNameService().page(null, 1000, pageIndex, null, null);
347 // if(pager.getRecords().isEmpty()){
350 // List<Registration> newRegs = new ArrayList<>(pager.getRecords().size());
351 // for(TaxonName name : pager.getRecords()){
355 // Set<String> extensionValues = name.getExtensions(getExtensionTypeIAPTRegData());
357 // // there is for sure only one
358 // if(extensionValues.isEmpty()){
362 // logger.debug("IAPT Registration for " + name.getTitleCache() + " ...");
364 // String iaptJson = extensionValues.iterator().next();
367 // IAPTRegData iaptData = new ObjectMapper().readValue(iaptJson, IAPTRegData.class);
369 // if(iaptData.getRegId() == null){
373 // DateTime regDate = null;
374 // if(iaptData.getDate() != null){
375 // DateTimeFormatter dateFormat;
376 // if(iaptData.getDate().matches("\\d{4}-\\d{2}-\\d{2}")){
377 // dateFormat = dateFormat2;
379 // dateFormat = dateFormat1;
382 // regDate = dateFormat.parseDateTime(iaptData.getDate());
383 // regDate.getYear();
384 // } catch (Exception e) {
385 // logger.error("Error parsing date : " + iaptData.getDate(), e);
390 // Registration reg = Registration.NewInstance();
391 // reg.setStatus(RegistrationStatus.PUBLISHED);
392 // reg.setIdentifier("http://phycobank.org/" + iaptData.getRegId());
393 // reg.setSpecificIdentifier(iaptData.getRegId().toString());
394 // reg.setInstitution(getInstitution(iaptData.getOffice()));
396 // boolean isPhycobankID = Integer.valueOf(reg.getSpecificIdentifier()) >= 100000;
398 // Partial youngestDate = null;
399 // Reference youngestPub = null;
401 // // find youngest publication
404 // // data imported from IAPT does not have typedesignation citations and sometimes no nomref
406 // if(isPhycobankID){
407 // youngestPub = name.getNomenclaturalReference();
408 // youngestDate = partial(youngestPub.getDatePublished());
410 // if(name.getTypeDesignations() != null && !name.getTypeDesignations().isEmpty()){
411 // for(TypeDesignationBase<?> td : name.getTypeDesignations()){
412 // if(td.getCitation() == null){
415 // Partial pubdate = partial(td.getCitation().getDatePublished());
416 // if(pubdate != null){
419 // if(youngestDate== null || earlierThanOther(youngestDate, pubdate)){
420 // youngestDate = pubdate;
421 // youngestPub = td.getCitation();
423 // } catch (Exception e) {
424 // logger.error("Error comparing " + youngestDate + " with" + pubdate , e);
431 // if((isPhycobankID && youngestPub == name.getNomenclaturalReference()) || !isPhycobankID) {
432 // reg.setName(name);
434 // logger.debug("skipping name published in older referece");
436 // if(name.getTypeDesignations() != null && !name.getTypeDesignations().isEmpty()){
437 // // do not add the collection directly to avoid "Found shared references to a collection" problem
438 // Set<TypeDesignationBase> typeDesignations = new HashSet<>(name.getTypeDesignations().size());
439 // for(TypeDesignationBase<?> td : name.getTypeDesignations()){
440 // if(td.getCitation() == null && isPhycobankID){
441 // logger.error("Missing TypeDesignation Citation in Phycobank data");
444 // if((isPhycobankID && youngestPub == td.getCitation()) || !isPhycobankID){
445 // typeDesignations.add(td);
447 // logger.debug("skipping typedesignation published in older reference");
450 // reg.setTypeDesignations(typeDesignations);
452 // reg.setRegistrationDate(regDate);
455 // } catch (JsonParseException e) {
456 // logger.error("Error parsing IAPTRegData from extension", e);
457 // } catch (JsonMappingException e) {
458 // logger.error("Error mapping json from extension to IAPTRegData", e);
459 // } catch (IOException e) {
464 // repo.getRegistrationService().save(newRegs);
466 // logger.debug("Registrations saved");
469 // repo.commitTransaction(tx);
474 private NomenclaturalStatusType
findILegitimateStatusType(TaxonName name
){
475 for(NomenclaturalStatus status
: name
.getStatus()){
476 if(status
.getType() != null && !status
.getType().isLegitimateType()){
477 return status
.getType();
485 // * @param youngestDate
489 // private boolean earlierThanOther(Partial basePartial, Partial other) {
491 // if(basePartial == null || basePartial.getValues().length == 0){
494 // if(other == null || other.getValues().length == 0){
497 // if(basePartial.size() == other.size()) {
498 // return basePartial.compareTo(other) < 0;
500 // basePartial = basePartial.without(DateTimeFieldType.dayOfMonth());
501 // other = other.without(DateTimeFieldType.dayOfMonth());
502 // if(basePartial.size() == other.size()) {
503 // return basePartial.compareTo(other) < 0;
505 // basePartial = basePartial.without(DateTimeFieldType.monthOfYear());
506 // other = other.without(DateTimeFieldType.monthOfYear());
507 // return basePartial.compareTo(other) < 0;
513 // * @param datePublished
516 // private Partial partial(TimePeriod datePublished) {
517 // if(datePublished != null){
518 // if(datePublished.getEnd() != null){
519 // return datePublished.getEnd();
521 // return datePublished.getStart();
532 // private Institution getInstitution(String office) {
533 // Institution institution;
534 // if(instituteMap.containsKey(office)){
535 // institution = instituteMap.get(office);
538 // Pager<Institution> pager = repo.getAgentService().findByTitleWithRestrictions(Institution.class, office, MatchMode.EXACT, null, null, null, null, null);
539 // ) if(!pager.getRecords().isEmpty()){
540 // institution = pager.getRecords().get(0);
542 // Institution institute = (Institution) repo.getAgentService().save(Institution.NewNamedInstance(office));
543 // institution = institute;
545 // instituteMap.put(office, institution);
547 // return institution;
551 // private ExtensionType getExtensionTypeIAPTRegData() {
552 // if(extensionTypeIAPTRegData == null){
553 // extensionTypeIAPTRegData = (ExtensionType) repo.getTermService().load(UUID.fromString("9be1bfe3-6ba0-4560-af15-86971ab96e09"));
555 // return extensionTypeIAPTRegData;