2 * Copyright (C) 2017 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
9 package eu
.etaxonomy
.cdm
.dataInserter
;
11 import java
.io
.IOException
;
12 import java
.util
.ArrayList
;
13 import java
.util
.Arrays
;
14 import java
.util
.EnumSet
;
15 import java
.util
.HashMap
;
16 import java
.util
.HashSet
;
17 import java
.util
.List
;
20 import java
.util
.UUID
;
22 import org
.apache
.log4j
.Logger
;
23 import org
.joda
.time
.DateTime
;
24 import org
.joda
.time
.DateTimeFieldType
;
25 import org
.joda
.time
.Partial
;
26 import org
.joda
.time
.format
.DateTimeFormatter
;
27 import org
.springframework
.context
.event
.ContextRefreshedEvent
;
28 import org
.springframework
.security
.authentication
.AuthenticationCredentialsNotFoundException
;
29 import org
.springframework
.security
.core
.GrantedAuthority
;
30 import org
.springframework
.transaction
.TransactionStatus
;
31 import org
.springframework
.transaction
.annotation
.Transactional
;
33 import com
.fasterxml
.jackson
.core
.JsonParseException
;
34 import com
.fasterxml
.jackson
.databind
.JsonMappingException
;
35 import com
.fasterxml
.jackson
.databind
.ObjectMapper
;
37 import eu
.etaxonomy
.cdm
.api
.application
.AbstractDataInserter
;
38 import eu
.etaxonomy
.cdm
.api
.application
.CdmRepository
;
39 import eu
.etaxonomy
.cdm
.api
.service
.pager
.Pager
;
40 import eu
.etaxonomy
.cdm
.model
.agent
.AgentBase
;
41 import eu
.etaxonomy
.cdm
.model
.agent
.Institution
;
42 import eu
.etaxonomy
.cdm
.model
.common
.ExtensionType
;
43 import eu
.etaxonomy
.cdm
.model
.common
.GrantedAuthorityImpl
;
44 import eu
.etaxonomy
.cdm
.model
.common
.Group
;
45 import eu
.etaxonomy
.cdm
.model
.common
.TimePeriod
;
46 import eu
.etaxonomy
.cdm
.model
.name
.Registration
;
47 import eu
.etaxonomy
.cdm
.model
.name
.RegistrationStatus
;
48 import eu
.etaxonomy
.cdm
.model
.name
.TaxonName
;
49 import eu
.etaxonomy
.cdm
.model
.name
.TypeDesignationBase
;
50 import eu
.etaxonomy
.cdm
.model
.reference
.Reference
;
51 import eu
.etaxonomy
.cdm
.persistence
.hibernate
.permission
.CRUD
;
52 import eu
.etaxonomy
.cdm
.persistence
.hibernate
.permission
.CdmAuthority
;
53 import eu
.etaxonomy
.cdm
.persistence
.hibernate
.permission
.CdmPermissionClass
;
54 import eu
.etaxonomy
.cdm
.persistence
.hibernate
.permission
.Role
;
55 import eu
.etaxonomy
.cdm
.persistence
.query
.MatchMode
;
56 import eu
.etaxonomy
.cdm
.vaadin
.model
.registration
.KindOfUnitTerms
;
57 import eu
.etaxonomy
.cdm
.vaadin
.permission
.RolesAndPermissions
;
61 * Can create missing registrations for names which have Extensions of the Type <code>IAPTRegdata.json</code>.
62 * See https://dev.e-taxonomy.eu/redmine/issues/6621 for further details.
63 * This feature can be activated by by supplying one of the following jvm command line arguments:
65 * <li><code>-DregistrationCreate=iapt</code>: create all iapt Registrations if missing</li>
66 * <li><code>-DregistrationWipeout=iapt</code>: remove all iapt Registrations</li>
67 * <li><code>-DregistrationWipeout=all</code>: remove all Registrations</li>
69 * The <code>-DregistrationWipeout</code> commands are executed before the <code>-DregistrationCreate</code> and will not change the name and type designations.
71 * @author a.kohlbecker
75 public class RegistrationRequiredDataInserter
extends AbstractDataInserter
{
77 protected static final String PARAM_NAME_CREATE
= "registrationCreate";
79 protected static final String PARAM_NAME_WIPEOUT
= "registrationWipeout";
81 protected static final UUID GROUP_SUBMITTER_UUID
= UUID
.fromString("c468c6a7-b96c-4206-849d-5a825f806d3e");
83 protected static final UUID GROUP_CURATOR_UUID
= UUID
.fromString("135210d3-3db7-4a81-ab36-240444637d45");
85 private static final EnumSet
<CRUD
> CREATE_READ
= EnumSet
.of(CRUD
.CREATE
, CRUD
.READ
);
86 private static final EnumSet
<CRUD
> CREATE_READ_UPDATE_DELETE
= EnumSet
.of(CRUD
.CREATE
, CRUD
.READ
, CRUD
.UPDATE
, CRUD
.DELETE
);
88 private static final Logger logger
= Logger
.getLogger(RegistrationRequiredDataInserter
.class);
90 private ExtensionType extensionTypeIAPTRegData
;
92 Map
<String
, Institution
> instituteMap
= new HashMap
<>();
94 public static boolean commandsExecuted
= false;
96 private CdmRepository repo
;
98 private boolean hasRun
= false;
100 public void setCdmRepository(CdmRepository repo
){
105 // ==================== Registration creation ======================= //
111 public void onApplicationEvent(ContextRefreshedEvent event
) {
117 runAsAuthentication(Role
.ROLE_ADMIN
);
119 insertRequiredData();
120 executeSuppliedCommands();
122 restoreAuthentication();
131 private void insertRequiredData() {
133 TransactionStatus txStatus
= repo
.startTransaction(false);
135 Role roleCuration
= RolesAndPermissions
.ROLE_CURATION
;
136 if(repo
.getGrantedAuthorityService().find(roleCuration
.getUuid()) == null){
137 repo
.getGrantedAuthorityService().saveOrUpdate(roleCuration
.asNewGrantedAuthority());
140 Group groupCurator
= repo
.getGroupService().load(GROUP_CURATOR_UUID
, Arrays
.asList("grantedAuthorities"));
141 if(groupCurator
== null){
142 groupCurator
= Group
.NewInstance();
143 groupCurator
.setUuid(GROUP_CURATOR_UUID
);
144 groupCurator
.setName("Curator");
146 assureGroupHas(groupCurator
, new CdmAuthority(CdmPermissionClass
.REGISTRATION
, CREATE_READ_UPDATE_DELETE
).toString());
147 repo
.getGroupService().saveOrUpdate(groupCurator
);
149 Group groupSubmitter
= repo
.getGroupService().load(GROUP_SUBMITTER_UUID
, Arrays
.asList("grantedAuthorities"));
150 if(groupSubmitter
== null){
151 groupSubmitter
= Group
.NewInstance();
152 groupSubmitter
.setUuid(GROUP_SUBMITTER_UUID
);
153 groupSubmitter
.setName("Submitter");
155 assureGroupHas(groupSubmitter
, new CdmAuthority(CdmPermissionClass
.TAXONNAME
, CREATE_READ
).toString());
156 assureGroupHas(groupSubmitter
, new CdmAuthority(CdmPermissionClass
.TEAMORPERSONBASE
, CREATE_READ
).toString());
157 assureGroupHas(groupSubmitter
, new CdmAuthority(CdmPermissionClass
.REGISTRATION
, CREATE_READ
).toString());
158 assureGroupHas(groupSubmitter
, new CdmAuthority(CdmPermissionClass
.REFERENCE
, CREATE_READ
).toString());
159 assureGroupHas(groupSubmitter
, new CdmAuthority(CdmPermissionClass
.SPECIMENOROBSERVATIONBASE
, CREATE_READ
).toString());
160 repo
.getGroupService().saveOrUpdate(groupSubmitter
);
162 if(repo
.getTermService().find(KindOfUnitTerms
.SPECIMEN().getUuid()) == null){
163 repo
.getTermService().save(KindOfUnitTerms
.SPECIMEN());
165 if(repo
.getTermService().find(KindOfUnitTerms
.PUBLISHED_IMAGE().getUuid()) == null){
166 repo
.getTermService().save(KindOfUnitTerms
.PUBLISHED_IMAGE());
168 if(repo
.getTermService().find(KindOfUnitTerms
.UNPUBLISHED_IMAGE().getUuid()) == null){
169 repo
.getTermService().save(KindOfUnitTerms
.UNPUBLISHED_IMAGE());
171 if(repo
.getTermService().find(KindOfUnitTerms
.CULTURE_METABOLIC_INACTIVE().getUuid()) == null){
172 repo
.getTermService().save(KindOfUnitTerms
.CULTURE_METABOLIC_INACTIVE());
175 // --- remove after release 4.12.0 ------------------------------------------------------
176 // delete old DerivationEventTypes terms which are no longer used, see #7059
177 // UUID_PUBLISHED_IMAGE = UUID.fromString("b8cba359-4202-4741-8ed8-4f17ae94b3e3");
178 // UUID UUID_UNPUBLISHED_IMAGE = UUID.fromString("6cd5681f-0918-4ed6-89a8-bda1480dc890");
179 // UUID UUID_CULTURE_METABOLIC_INACTIVE = UUID.fromString("eaf1c853-ba8d-4c40-aa0a-56beac96b0d2");
180 for(UUID uuid
: new UUID
[]{
181 UUID
.fromString("b8cba359-4202-4741-8ed8-4f17ae94b3e3"),
182 UUID
.fromString("6cd5681f-0918-4ed6-89a8-bda1480dc890"),
183 UUID
.fromString("eaf1c853-ba8d-4c40-aa0a-56beac96b0d2")}){
184 if(repo
.getTermService().find(uuid
) != null){
185 repo
.getTermService().delete(uuid
);
188 // --------------------------------------------------------------------------------------
190 repo
.commitTransaction(txStatus
);
194 private void assureGroupHas(Group group
, String authorityString
){
195 boolean authorityExists
= false;
197 for(GrantedAuthority ga
: group
.getGrantedAuthorities()){
198 if((authorityExists
= ga
.getAuthority().equals(authorityString
)) == true){
202 if(!authorityExists
){
203 group
.addGrantedAuthority(findGrantedAuthority(authorityString
));
207 private GrantedAuthorityImpl
findGrantedAuthority(String authorityString
){
208 GrantedAuthorityImpl ga
= null;
210 ga
= repo
.getGrantedAuthorityService().findAuthorityString(authorityString
);
211 } catch (AuthenticationCredentialsNotFoundException e
){
215 ga
= GrantedAuthorityImpl
.NewInstance(authorityString
);
216 repo
.getGrantedAuthorityService().save(ga
);
225 private void executeSuppliedCommands() {
227 if(commandsExecuted
){
229 // a second run could take place during initialization of the web context
232 commandsExecuted
= true;
234 String wipeoutCmd
= System
.getProperty(PARAM_NAME_WIPEOUT
);
235 String createCmd
= System
.getProperty(PARAM_NAME_CREATE
);
237 // ============ DELETE
238 if(wipeoutCmd
!= null && wipeoutCmd
.matches("iapt|all")){
240 boolean onlyIapt
= wipeoutCmd
.equals("iapt");
241 Set
<UUID
> deleteCandidates
= new HashSet
<UUID
>();
243 TransactionStatus tx
= repo
.startTransaction(true);
244 List
<Registration
> allRegs
= repo
.getRegistrationService().list(null, null, null, null, null);
245 for(Registration reg
: allRegs
){
248 @SuppressWarnings("unchecked")
249 Set
<String
> extensions
= reg
.getName().getExtensions(getExtensionTypeIAPTRegData());
250 if(reg
.getUuid() != null){
251 deleteCandidates
.add(reg
.getUuid());
253 } catch(NullPointerException e
){
257 if(reg
.getUuid() != null){
258 deleteCandidates
.add(reg
.getUuid());
262 repo
.commitTransaction(tx
);
263 if(!deleteCandidates
.isEmpty()){
265 repo
.getRegistrationService().delete(deleteCandidates
);
266 } catch (Exception e
) {
267 // MySQLIntegrityConstraintViolationException happens here every second run !!!
273 // ============ CREATE
275 if(createCmd
!= null && createCmd
.equals("iapt")){
277 DateTimeFormatter dateFormat1
= org
.joda
.time
.format
.DateTimeFormat
.forPattern("dd.MM.yy").withPivotYear(1950);
278 DateTimeFormatter dateFormat2
= org
.joda
.time
.format
.DateTimeFormat
.forPattern("yyyy-MM-dd").withPivotYear(1950);
280 TransactionStatus tx
= repo
.startTransaction(false);
282 Pager
<TaxonName
> pager
= repo
.getNameService().page(null, 1000, pageIndex
, null, null);
283 if(pager
.getRecords().isEmpty()){
286 List
<Registration
> newRegs
= new ArrayList
<>(pager
.getRecords().size());
287 for(TaxonName name
: pager
.getRecords()){
291 Set
<String
> extensionValues
= name
.getExtensions(getExtensionTypeIAPTRegData());
293 // there is for sure only one
294 if(extensionValues
.isEmpty()){
298 logger
.debug("IAPT Registration for " + name
.getTitleCache() + " ...");
300 String iaptJson
= extensionValues
.iterator().next();
303 IAPTRegData iaptData
= new ObjectMapper().readValue(iaptJson
, IAPTRegData
.class);
305 if(iaptData
.getRegId() == null){
309 DateTime regDate
= null;
310 if(iaptData
.getDate() != null){
311 DateTimeFormatter dateFormat
;
312 if(iaptData
.getDate().matches("\\d{4}-\\d{2}-\\d{2}")){
313 dateFormat
= dateFormat2
;
315 dateFormat
= dateFormat1
;
318 regDate
= dateFormat
.parseDateTime(iaptData
.getDate());
320 } catch (Exception e
) {
321 logger
.error("Error parsing date : " + iaptData
.getDate(), e
);
326 Registration reg
= Registration
.NewInstance();
327 reg
.setStatus(RegistrationStatus
.PUBLISHED
);
328 reg
.setIdentifier("http://phycobank.org/" + iaptData
.getRegId());
329 reg
.setSpecificIdentifier(iaptData
.getRegId().toString());
330 reg
.setInstitution(getInstitution(iaptData
.getOffice()));
332 boolean isPhycobankID
= Integer
.valueOf(reg
.getSpecificIdentifier()) >= 100000;
334 Partial youngestDate
= null;
335 Reference youngestPub
= null;
337 // find youngest publication
340 // data imported from IAPT does not have typedesignation citations and sometimes no nomref
343 youngestPub
= name
.getNomenclaturalReference();
344 youngestDate
= partial(youngestPub
.getDatePublished());
346 if(name
.getTypeDesignations() != null && !name
.getTypeDesignations().isEmpty()){
347 for(TypeDesignationBase
<?
> td
: name
.getTypeDesignations()){
348 if(td
.getCitation() == null){
351 Partial pubdate
= partial(td
.getCitation().getDatePublished());
353 if(youngestDate
== null || comparePartials(youngestDate
, pubdate
)){
354 youngestDate
= pubdate
;
355 youngestPub
= td
.getCitation();
362 if((isPhycobankID
&& youngestPub
== name
.getNomenclaturalReference()) || !isPhycobankID
) {
365 logger
.debug("skipping name published in older referece");
367 if(name
.getTypeDesignations() != null && !name
.getTypeDesignations().isEmpty()){
368 // do not add the collection directly to avoid "Found shared references to a collection" problem
369 Set
<TypeDesignationBase
> typeDesignations
= new HashSet
<>(name
.getTypeDesignations().size());
370 for(TypeDesignationBase
<?
> td
: name
.getTypeDesignations()){
371 if(td
.getCitation() == null && isPhycobankID
){
372 logger
.error("Missing TypeDesignation Citation in Phycobank data");
375 if((isPhycobankID
&& youngestPub
== td
.getCitation()) || !isPhycobankID
){
376 typeDesignations
.add(td
);
378 logger
.debug("skipping typedesignation published in older reference");
381 reg
.setTypeDesignations(typeDesignations
);
383 reg
.setRegistrationDate(regDate
);
386 } catch (JsonParseException e
) {
387 logger
.error("Error parsing IAPTRegData from extension", e
);
388 } catch (JsonMappingException e
) {
389 logger
.error("Error mapping json from extension to IAPTRegData", e
);
390 } catch (IOException e
) {
395 repo
.getRegistrationService().save(newRegs
);
397 logger
.debug("Registrations saved");
400 repo
.commitTransaction(tx
);
406 * @param youngestDate
410 protected boolean comparePartials(Partial youngestDate
, Partial pubdate
) {
412 if(youngestDate
.size() == pubdate
.size()) {
413 return youngestDate
.compareTo(pubdate
) < 0;
415 youngestDate
= youngestDate
.without(DateTimeFieldType
.dayOfMonth());
416 pubdate
= pubdate
.without(DateTimeFieldType
.dayOfMonth());
417 if(youngestDate
.size() == pubdate
.size()) {
418 return youngestDate
.compareTo(pubdate
) < 0;
420 youngestDate
= youngestDate
.without(DateTimeFieldType
.monthOfYear());
421 pubdate
= pubdate
.without(DateTimeFieldType
.monthOfYear());
422 return youngestDate
.compareTo(pubdate
) < 0;
428 * @param datePublished
431 private Partial
partial(TimePeriod datePublished
) {
432 if(datePublished
!= null){
433 if(datePublished
.getEnd() != null){
434 return datePublished
.getEnd();
436 return datePublished
.getStart();
447 private Institution
getInstitution(String office
) {
448 Institution institution
;
449 if(instituteMap
.containsKey(office
)){
450 institution
= instituteMap
.get(office
);
453 Pager
<AgentBase
> pager
= repo
.getAgentService().findByTitle(Institution
.class, office
, MatchMode
.EXACT
, null, null, null, null, null);
454 if(!pager
.getRecords().isEmpty()){
455 institution
= (Institution
) pager
.getRecords().get(0);
457 Institution institute
= (Institution
) repo
.getAgentService().save(Institution
.NewNamedInstance(office
));
458 institution
= institute
;
460 instituteMap
.put(office
, institution
);
466 private ExtensionType
getExtensionTypeIAPTRegData() {
467 if(extensionTypeIAPTRegData
== null){
468 extensionTypeIAPTRegData
= (ExtensionType
) repo
.getTermService().load(UUID
.fromString("9be1bfe3-6ba0-4560-af15-86971ab96e09"));
470 return extensionTypeIAPTRegData
;