update vaadin to v5.18.5
[cdm-vaadin.git] / src / main / java / eu / etaxonomy / cdm / dataInserter / RegistrationRequiredDataInserter.java
1 /**
2 * Copyright (C) 2017 EDIT
3 * European Distributed Institute of Taxonomy
4 * http://www.e-taxonomy.eu
5 *
6 * The contents of this file are subject to the Mozilla Public License Version 1.1
7 * See LICENSE.TXT at the top of this package for the full license terms.
8 */
9 package eu.etaxonomy.cdm.dataInserter;
10
11 import java.util.ArrayList;
12 import java.util.Arrays;
13 import java.util.EnumSet;
14 import java.util.HashMap;
15 import java.util.List;
16 import java.util.Map;
17 import java.util.Set;
18 import java.util.UUID;
19
20 import org.apache.log4j.Level;
21 import org.apache.log4j.Logger;
22 import org.hibernate.Session;
23 import org.springframework.context.event.ContextRefreshedEvent;
24 import org.springframework.security.authentication.AuthenticationCredentialsNotFoundException;
25 import org.springframework.security.core.GrantedAuthority;
26 import org.springframework.transaction.TransactionStatus;
27 import org.springframework.transaction.annotation.Transactional;
28
29 import eu.etaxonomy.cdm.api.application.AbstractDataInserter;
30 import eu.etaxonomy.cdm.api.application.CdmRepository;
31 import eu.etaxonomy.cdm.api.service.pager.Pager;
32 import eu.etaxonomy.cdm.model.agent.Institution;
33 import eu.etaxonomy.cdm.model.name.NomenclaturalStatus;
34 import eu.etaxonomy.cdm.model.name.NomenclaturalStatusType;
35 import eu.etaxonomy.cdm.model.name.Rank;
36 import eu.etaxonomy.cdm.model.name.TaxonName;
37 import eu.etaxonomy.cdm.model.permission.CRUD;
38 import eu.etaxonomy.cdm.model.permission.GrantedAuthorityImpl;
39 import eu.etaxonomy.cdm.model.permission.Group;
40 import eu.etaxonomy.cdm.model.permission.PermissionClass;
41 import eu.etaxonomy.cdm.model.taxon.Taxon;
42 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
43 import eu.etaxonomy.cdm.model.term.DefinedTerm;
44 import eu.etaxonomy.cdm.model.term.DefinedTermBase;
45 import eu.etaxonomy.cdm.model.term.TermVocabulary;
46 import eu.etaxonomy.cdm.persistence.dao.hibernate.taxonGraph.AbstractHibernateTaxonGraphProcessor;
47 import eu.etaxonomy.cdm.persistence.dao.taxonGraph.TaxonGraphException;
48 import eu.etaxonomy.cdm.persistence.permission.CdmAuthority;
49 import eu.etaxonomy.cdm.persistence.permission.Role;
50 import eu.etaxonomy.cdm.vaadin.model.registration.KindOfUnitTerms;
51 import eu.etaxonomy.cdm.vaadin.permission.RolesAndPermissions;
52
53 ///*
54 // * Can create missing registrations for names which have Extensions of the Type <code>IAPTRegdata.json</code>.
55 //* See https://dev.e-taxonomy.eu/redmine/issues/6621 for further details.
56 //* This feature can be activated by by supplying one of the following jvm command line arguments:
57 //* <ul>
58 //* <li><code>-DregistrationCreate=iapt</code>: create all iapt Registrations if missing</li>
59 //* <li><code>-DregistrationWipeout=iapt</code>: remove all iapt Registrations</li>
60 //* <li><code>-DregistrationWipeout=all</code>: remove all Registrations</li>
61 //* </ul>
62 //* The <code>-DregistrationWipeout</code> commands are executed before the <code>-DregistrationCreate</code> and will not change the name and type designations.
63 //*/
64 /**
65 * This feature can be activated by by supplying one of the following jvm command line arguments:
66 * <ul>
67 * <li><code>-DtaxonGraphCreate=true</code>: create taxon graph relations for all names below genus level</li>
68 * </ul>
69 *
70 * @author a.kohlbecker
71 * @since May 9, 2017
72 */
73 public class RegistrationRequiredDataInserter extends AbstractDataInserter {
74
75 // protected static final String PARAM_NAME_CREATE = "registrationCreate";
76 //
77 // protected static final String PARAM_NAME_WIPEOUT = "registrationWipeout";
78
79 protected static final String TAXON_GRAPH_CREATE = "taxonGraphCreate";
80
81 protected static final UUID GROUP_SUBMITTER_UUID = UUID.fromString("c468c6a7-b96c-4206-849d-5a825f806d3e");
82
83 protected static final UUID GROUP_CURATOR_UUID = UUID.fromString("135210d3-3db7-4a81-ab36-240444637d45");
84
85 private static final EnumSet<CRUD> CREATE_READ = EnumSet.of(CRUD.CREATE, CRUD.READ);
86 private static final EnumSet<CRUD> CREATE_READ_UPDATE_DELETE = EnumSet.of(CRUD.CREATE, CRUD.READ, CRUD.UPDATE, CRUD.DELETE);
87
88 private static final Logger logger = Logger.getLogger(RegistrationRequiredDataInserter.class);
89
90 // private ExtensionType extensionTypeIAPTRegData;
91
92 Map<String, Institution> instituteMap = new HashMap<>();
93
94 public static boolean commandsExecuted = false;
95
96 private CdmRepository repo;
97
98 private boolean hasRun = false;
99
100 public void setCdmRepository(CdmRepository repo){
101 this.repo = repo;
102 }
103
104 @Override
105 public void onApplicationEvent(ContextRefreshedEvent event) {
106
107 if(hasRun){
108 return;
109 }
110
111 runAsAuthentication(Role.ROLE_ADMIN);
112
113 insertRequiredData();
114 executeSuppliedCommands();
115
116 restoreAuthentication();
117
118 hasRun = true;
119 }
120
121 @Transactional
122 private void insertRequiredData() {
123
124 TransactionStatus txStatus = repo.startTransaction(false);
125
126 Role roleCuration = RolesAndPermissions.ROLE_CURATION;
127 if(repo.getGrantedAuthorityService().find(roleCuration.getUuid()) == null){
128 repo.getGrantedAuthorityService().saveOrUpdate(roleCuration.asNewGrantedAuthority());
129 }
130
131 Group groupCurator = repo.getGroupService().load(GROUP_CURATOR_UUID, Arrays.asList("grantedAuthorities"));
132 if(groupCurator == null){
133 groupCurator = Group.NewInstance();
134 groupCurator.setUuid(GROUP_CURATOR_UUID);
135 groupCurator.setName("Curator");
136 }
137 assureGroupHas(groupCurator, new CdmAuthority(PermissionClass.REGISTRATION, CREATE_READ_UPDATE_DELETE).toString());
138 repo.getGroupService().saveOrUpdate(groupCurator);
139
140 Group groupSubmitter = repo.getGroupService().load(GROUP_SUBMITTER_UUID, Arrays.asList("grantedAuthorities"));
141 if(groupSubmitter == null){
142 groupSubmitter = Group.NewInstance();
143 groupSubmitter.setUuid(GROUP_SUBMITTER_UUID);
144 groupSubmitter.setName("Submitter");
145 }
146 assureGroupHas(groupSubmitter, new CdmAuthority(PermissionClass.TAXONNAME, CREATE_READ).toString());
147 assureGroupHas(groupSubmitter, new CdmAuthority(PermissionClass.TEAMORPERSONBASE, CREATE_READ).toString());
148 assureGroupHas(groupSubmitter, new CdmAuthority(PermissionClass.REGISTRATION, CREATE_READ).toString());
149 assureGroupHas(groupSubmitter, new CdmAuthority(PermissionClass.REFERENCE, CREATE_READ).toString());
150 assureGroupHas(groupSubmitter, new CdmAuthority(PermissionClass.SPECIMENOROBSERVATIONBASE, CREATE_READ).toString());
151 assureGroupHas(groupSubmitter, new CdmAuthority(PermissionClass.COLLECTION, CREATE_READ).toString());
152 repo.getGroupService().saveOrUpdate(groupSubmitter);
153
154 TermVocabulary<DefinedTerm> kindOfUnitVocabulary = repo.getVocabularyService().find(KindOfUnitTerms.KIND_OF_UNIT_VOCABULARY().getUuid());
155 if(kindOfUnitVocabulary == null){
156 kindOfUnitVocabulary = repo.getVocabularyService().save(KindOfUnitTerms.KIND_OF_UNIT_VOCABULARY());
157 }
158
159 DefinedTermBase<?> kouSpecimen = repo.getTermService().find(KindOfUnitTerms.SPECIMEN().getUuid());
160 DefinedTermBase<?> kouImage = repo.getTermService().find(KindOfUnitTerms.PUBLISHED_IMAGE().getUuid());
161 DefinedTermBase<?> kouUnpublishedImage = repo.getTermService().find(KindOfUnitTerms.UNPUBLISHED_IMAGE().getUuid());
162 DefinedTermBase<?> kouCulture = repo.getTermService().find(KindOfUnitTerms.CULTURE_METABOLIC_INACTIVE().getUuid());
163
164 if(kouSpecimen == null){
165 kouSpecimen = repo.getTermService().save(KindOfUnitTerms.SPECIMEN());
166 }
167 if(kouImage == null){
168 kouImage = repo.getTermService().save(KindOfUnitTerms.PUBLISHED_IMAGE());
169 }
170 if(kouUnpublishedImage == null){
171 kouUnpublishedImage = repo.getTermService().save(KindOfUnitTerms.UNPUBLISHED_IMAGE());
172 }
173 if(kouCulture == null){
174 kouCulture = repo.getTermService().save(KindOfUnitTerms.CULTURE_METABOLIC_INACTIVE());
175 }
176
177 Set<DefinedTerm> termInVocab = kindOfUnitVocabulary.getTerms();
178 List<DefinedTermBase> kouTerms = Arrays.asList(kouCulture, kouImage, kouSpecimen, kouUnpublishedImage);
179
180 for(DefinedTermBase<?> t : kouTerms){
181 if(!termInVocab.contains(t)){
182 kindOfUnitVocabulary.addTerm((DefinedTerm)t);
183 }
184 }
185
186 repo.commitTransaction(txStatus);
187
188 }
189
190 private void assureGroupHas(Group group, String authorityString){
191 boolean authorityExists = false;
192
193 for(GrantedAuthority ga : group.getGrantedAuthorities()){
194 if((authorityExists = ga.getAuthority().equals(authorityString)) == true){
195 break;
196 }
197 }
198 if(!authorityExists){
199 group.addGrantedAuthority(findGrantedAuthority(authorityString));
200 }
201 }
202
203 private GrantedAuthorityImpl findGrantedAuthority(String authorityString){
204 GrantedAuthorityImpl ga = null;
205 try{
206 ga = repo.getGrantedAuthorityService().findAuthorityString(authorityString);
207 } catch (AuthenticationCredentialsNotFoundException e){
208 e.printStackTrace();
209 }
210 if(ga == null){
211 ga = GrantedAuthorityImpl.NewInstance(authorityString);
212 repo.getGrantedAuthorityService().save(ga);
213 }
214 return ga;
215 }
216
217 /**
218 *
219 */
220 private void executeSuppliedCommands() {
221
222 if(commandsExecuted){
223 // do not run twice
224 // a second run could take place during initialization of the web context
225 return;
226 }
227 commandsExecuted = true;
228
229 String taxonGraphCreate = System.getProperty(TAXON_GRAPH_CREATE);
230
231 if(taxonGraphCreate != null){
232 AbstractHibernateTaxonGraphProcessor processor = new AbstractHibernateTaxonGraphProcessor() {
233
234 @Override
235 public Session getSession() {
236 return repo.getSession();
237 }
238 };
239 logger.setLevel(Level.DEBUG);
240 int chunksize = 1000;
241 int pageIndex = 0;
242 TransactionStatus tx;
243 Pager<Taxon> taxonPage;
244 List<TaxonBase> taxa = new ArrayList<>();
245 logger.debug("======= fixing sec refrences =========");
246 while(true){
247 tx = repo.startTransaction(false);
248 taxonPage = repo.getTaxonService().page(Taxon.class, chunksize, pageIndex++, null, null);
249 if(taxonPage.getRecords().size() == 0){
250 repo.commitTransaction(tx);
251 break;
252 }
253 for(Taxon taxon : taxonPage.getRecords()){
254 taxon.setSec(processor.secReference());
255 repo.getTaxonService().saveOrUpdate(taxon);
256 }
257 repo.commitTransaction(tx);
258 }
259
260 logger.debug("======= creating taxon graph =========");
261 pageIndex = 0;
262 Pager<TaxonName> page;
263 while(true){
264 tx = repo.startTransaction(false);
265 page = repo.getNameService().page(null, chunksize, pageIndex++, null, null);
266 if(page.getRecords().size() == 0){
267 repo.commitTransaction(tx);
268 break;
269 }
270 logger.debug(TAXON_GRAPH_CREATE + ": chunk " + pageIndex + "/" + Math.ceil(page.getCount() / chunksize));
271 taxa = new ArrayList<>();
272
273 for(TaxonName name : page.getRecords()){
274 if(name.getRank() != null && name.getRank().isLower(Rank.GENUS())){
275 NomenclaturalStatusType illegitimType = findILegitimateStatusType(name);
276 if(illegitimType == null){
277 Taxon taxon;
278 try {
279 logger.debug("Processing name: " + name.getTitleCache() + " [" + name.getRank().getLabel() + "]");
280 taxon = processor.assureSingleTaxon(name);
281 processor.updateEdges(taxon);
282 taxa.add(taxon);
283 } catch (TaxonGraphException e) {
284 logger.error(e.getMessage());
285 }
286 } else {
287 logger.debug("Skipping illegitimate name: " + name.getTitleCache() + " " + illegitimType.getLabel() + " [" + name.getRank().getLabel() + "]");
288 }
289 } else {
290 logger.debug("Skipping name: " + name.getTitleCache() + " [" + (name.getRank() != null ? name.getRank().getLabel() : "NULL") + "]");
291 }
292 }
293 repo.getTaxonService().saveOrUpdate(taxa);
294 repo.commitTransaction(tx);
295 }
296 }
297
298 // String wipeoutCmd = System.getProperty(PARAM_NAME_WIPEOUT);
299 // String createCmd = System.getProperty(PARAM_NAME_CREATE);
300 //
301 // // ============ DELETE
302 // if(wipeoutCmd != null && wipeoutCmd.matches("iapt|all")){
303 //
304 // boolean onlyIapt = wipeoutCmd.equals("iapt");
305 // Set<UUID> deleteCandidates = new HashSet<UUID>();
306 //
307 // TransactionStatus tx = repo.startTransaction(true);
308 // List<Registration> allRegs = repo.getRegistrationService().list(null, null, null, null, null);
309 // for(Registration reg : allRegs){
310 // if(onlyIapt){
311 // try {
312 // @SuppressWarnings("unchecked")
313 // Set<String> extensions = reg.getName().getExtensions(getExtensionTypeIAPTRegData());
314 // if(reg.getUuid() != null){
315 // deleteCandidates.add(reg.getUuid());
316 // }
317 // } catch(NullPointerException e){
318 // // IGNORE
319 // }
320 // } else {
321 // if(reg.getUuid() != null){
322 // deleteCandidates.add(reg.getUuid());
323 // }
324 // }
325 // }
326 // repo.commitTransaction(tx);
327 // if(!deleteCandidates.isEmpty()){
328 // try {
329 // repo.getRegistrationService().delete(deleteCandidates);
330 // } catch (Exception e) {
331 // // MySQLIntegrityConstraintViolationException happens here every second run !!!
332 // logger.error(e);
333 // }
334 // }
335 // }
336 //
337 // // ============ CREATE
338 // int pageIndex = 0;
339 // if(createCmd != null && createCmd.equals("iapt")){
340 //
341 // DateTimeFormatter dateFormat1 = org.joda.time.format.DateTimeFormat.forPattern("dd.MM.yy").withPivotYear(1950);
342 // DateTimeFormatter dateFormat2 = org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd").withPivotYear(1950);
343 //
344 // TransactionStatus tx = repo.startTransaction(false);
345 // while(true) {
346 // Pager<TaxonName> pager = repo.getNameService().page(null, 1000, pageIndex, null, null);
347 // if(pager.getRecords().isEmpty()){
348 // break;
349 // }
350 // List<Registration> newRegs = new ArrayList<>(pager.getRecords().size());
351 // for(TaxonName name : pager.getRecords()){
352 //
353 //
354 //
355 // Set<String> extensionValues = name.getExtensions(getExtensionTypeIAPTRegData());
356 //
357 // // there is for sure only one
358 // if(extensionValues.isEmpty()){
359 // continue;
360 // }
361 //
362 // logger.debug("IAPT Registration for " + name.getTitleCache() + " ...");
363 //
364 // String iaptJson = extensionValues.iterator().next();
365 // try {
366 //
367 // IAPTRegData iaptData = new ObjectMapper().readValue(iaptJson, IAPTRegData.class);
368 //
369 // if(iaptData.getRegId() == null){
370 // continue;
371 // }
372 //
373 // DateTime regDate = null;
374 // if(iaptData.getDate() != null){
375 // DateTimeFormatter dateFormat;
376 // if(iaptData.getDate().matches("\\d{4}-\\d{2}-\\d{2}")){
377 // dateFormat = dateFormat2;
378 // } else {
379 // dateFormat = dateFormat1;
380 // }
381 // try {
382 // regDate = dateFormat.parseDateTime(iaptData.getDate());
383 // regDate.getYear();
384 // } catch (Exception e) {
385 // logger.error("Error parsing date : " + iaptData.getDate(), e);
386 // continue;
387 // }
388 // }
389 //
390 // Registration reg = Registration.NewInstance();
391 // reg.setStatus(RegistrationStatus.PUBLISHED);
392 // reg.setIdentifier("http://phycobank.org/" + iaptData.getRegId());
393 // reg.setSpecificIdentifier(iaptData.getRegId().toString());
394 // reg.setInstitution(getInstitution(iaptData.getOffice()));
395 //
396 // boolean isPhycobankID = Integer.valueOf(reg.getSpecificIdentifier()) >= 100000;
397 //
398 // Partial youngestDate = null;
399 // Reference youngestPub = null;
400 //
401 // // find youngest publication
402 //
403 // // NOTE:
404 // // data imported from IAPT does not have typedesignation citations and sometimes no nomref
405 //
406 // if(isPhycobankID){
407 // youngestPub = name.getNomenclaturalReference();
408 // youngestDate = partial(youngestPub.getDatePublished());
409 //
410 // if(name.getTypeDesignations() != null && !name.getTypeDesignations().isEmpty()){
411 // for(TypeDesignationBase<?> td : name.getTypeDesignations()){
412 // if(td.getCitation() == null){
413 // continue;
414 // }
415 // Partial pubdate = partial(td.getCitation().getDatePublished());
416 // if(pubdate != null){
417 //
418 // try {
419 // if(youngestDate== null || earlierThanOther(youngestDate, pubdate)){
420 // youngestDate = pubdate;
421 // youngestPub = td.getCitation();
422 // }
423 // } catch (Exception e) {
424 // logger.error("Error comparing " + youngestDate + " with" + pubdate , e);
425 // }
426 // }
427 // }
428 // }
429 // }
430 //
431 // if((isPhycobankID && youngestPub == name.getNomenclaturalReference()) || !isPhycobankID) {
432 // reg.setName(name);
433 // } else {
434 // logger.debug("skipping name published in older referece");
435 // }
436 // if(name.getTypeDesignations() != null && !name.getTypeDesignations().isEmpty()){
437 // // do not add the collection directly to avoid "Found shared references to a collection" problem
438 // Set<TypeDesignationBase> typeDesignations = new HashSet<>(name.getTypeDesignations().size());
439 // for(TypeDesignationBase<?> td : name.getTypeDesignations()){
440 // if(td.getCitation() == null && isPhycobankID){
441 // logger.error("Missing TypeDesignation Citation in Phycobank data");
442 // continue;
443 // }
444 // if((isPhycobankID && youngestPub == td.getCitation()) || !isPhycobankID){
445 // typeDesignations.add(td);
446 // } else {
447 // logger.debug("skipping typedesignation published in older reference");
448 // }
449 // }
450 // reg.setTypeDesignations(typeDesignations);
451 // }
452 // reg.setRegistrationDate(regDate);
453 // newRegs.add(reg);
454 //
455 // } catch (JsonParseException e) {
456 // logger.error("Error parsing IAPTRegData from extension", e);
457 // } catch (JsonMappingException e) {
458 // logger.error("Error mapping json from extension to IAPTRegData", e);
459 // } catch (IOException e) {
460 // logger.error(e);
461 // }
462 //
463 // }
464 // repo.getRegistrationService().save(newRegs);
465 // tx.flush();
466 // logger.debug("Registrations saved");
467 // pageIndex++;
468 // }
469 // repo.commitTransaction(tx);
470 // }
471
472 }
473
474 private NomenclaturalStatusType findILegitimateStatusType(TaxonName name){
475 for(NomenclaturalStatus status : name.getStatus()){
476 if(status.getType() != null && !status.getType().isLegitimate()){
477 return status.getType();
478 }
479 }
480 return null;
481 }
482
483
484 // /**
485 // * @param youngestDate
486 // * @param pubdate
487 // * @return
488 // */
489 // private boolean earlierThanOther(Partial basePartial, Partial other) {
490 //
491 // if(basePartial == null || basePartial.getValues().length == 0){
492 // return false;
493 // }
494 // if(other == null || other.getValues().length == 0){
495 // return true;
496 // }
497 // if(basePartial.size() == other.size()) {
498 // return basePartial.compareTo(other) < 0;
499 // }
500 // basePartial = basePartial.without(DateTimeFieldType.dayOfMonth());
501 // other = other.without(DateTimeFieldType.dayOfMonth());
502 // if(basePartial.size() == other.size()) {
503 // return basePartial.compareTo(other) < 0;
504 // }
505 // basePartial = basePartial.without(DateTimeFieldType.monthOfYear());
506 // other = other.without(DateTimeFieldType.monthOfYear());
507 // return basePartial.compareTo(other) < 0;
508 //
509 // }
510
511
512 // /**
513 // * @param datePublished
514 // * @return
515 // */
516 // private Partial partial(TimePeriod datePublished) {
517 // if(datePublished != null){
518 // if(datePublished.getEnd() != null){
519 // return datePublished.getEnd();
520 // } else {
521 // return datePublished.getStart();
522 // }
523 // }
524 // return null;
525 // }
526
527
528 // /**
529 // * @param office
530 // * @return
531 // */
532 // private Institution getInstitution(String office) {
533 // Institution institution;
534 // if(instituteMap.containsKey(office)){
535 // institution = instituteMap.get(office);
536 // } else {
537 //
538 // Pager<Institution> pager = repo.getAgentService().findByTitleWithRestrictions(Institution.class, office, MatchMode.EXACT, null, null, null, null, null);
539 // ) if(!pager.getRecords().isEmpty()){
540 // institution = pager.getRecords().get(0);
541 // } else {
542 // Institution institute = (Institution) repo.getAgentService().save(Institution.NewNamedInstance(office));
543 // institution = institute;
544 // }
545 // instituteMap.put(office, institution);
546 // }
547 // return institution;
548 // }
549
550
551 // private ExtensionType getExtensionTypeIAPTRegData() {
552 // if(extensionTypeIAPTRegData == null){
553 // extensionTypeIAPTRegData = (ExtensionType) repo.getTermService().load(UUID.fromString("9be1bfe3-6ba0-4560-af15-86971ab96e09"));
554 // }
555 // return extensionTypeIAPTRegData;
556 // }
557
558
559 }