src/main/java/eu/etaxonomy/cdm/dataInserter/RegistrationRequiredDataInserter.java

   1 /**
   2 * Copyright (C) 2017 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.dataInserter;
  10
  11 import java.io.IOException;
  12 import java.util.ArrayList;
  13 import java.util.Arrays;
  14 import java.util.EnumSet;
  15 import java.util.HashMap;
  16 import java.util.HashSet;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.Set;
  20 import java.util.UUID;
  21
  22 import org.apache.log4j.Logger;
  23 import org.joda.time.DateTime;
  24 import org.joda.time.DateTimeFieldType;
  25 import org.joda.time.Partial;
  26 import org.joda.time.format.DateTimeFormatter;
  27 import org.springframework.context.event.ContextRefreshedEvent;
  28 import org.springframework.security.authentication.AuthenticationCredentialsNotFoundException;
  29 import org.springframework.security.core.GrantedAuthority;
  30 import org.springframework.transaction.TransactionStatus;
  31 import org.springframework.transaction.annotation.Transactional;
  32
  33 import com.fasterxml.jackson.core.JsonParseException;
  34 import com.fasterxml.jackson.databind.JsonMappingException;
  35 import com.fasterxml.jackson.databind.ObjectMapper;
  36
  37 import eu.etaxonomy.cdm.api.application.AbstractDataInserter;
  38 import eu.etaxonomy.cdm.api.application.CdmRepository;
  39 import eu.etaxonomy.cdm.api.service.pager.Pager;
  40 import eu.etaxonomy.cdm.model.agent.AgentBase;
  41 import eu.etaxonomy.cdm.model.agent.Institution;
  42 import eu.etaxonomy.cdm.model.common.ExtensionType;
  43 import eu.etaxonomy.cdm.model.common.GrantedAuthorityImpl;
  44 import eu.etaxonomy.cdm.model.common.Group;
  45 import eu.etaxonomy.cdm.model.common.TimePeriod;
  46 import eu.etaxonomy.cdm.model.name.Registration;
  47 import eu.etaxonomy.cdm.model.name.RegistrationStatus;
  48 import eu.etaxonomy.cdm.model.name.TaxonName;
  49 import eu.etaxonomy.cdm.model.name.TypeDesignationBase;
  50 import eu.etaxonomy.cdm.model.reference.Reference;
  51 import eu.etaxonomy.cdm.persistence.hibernate.permission.CRUD;
  52 import eu.etaxonomy.cdm.persistence.hibernate.permission.CdmAuthority;
  53 import eu.etaxonomy.cdm.persistence.hibernate.permission.CdmPermissionClass;
  54 import eu.etaxonomy.cdm.persistence.hibernate.permission.Role;
  55 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  56 import eu.etaxonomy.cdm.vaadin.model.registration.KindOfUnitTerms;
  57 import eu.etaxonomy.cdm.vaadin.security.RolesAndPermissions;
  58
  59 /**
  60  *
  61  * Can create missing registrations for names which have Extensions of the Type <code>IAPTRegdata.json</code>.
  62  * See https://dev.e-taxonomy.eu/redmine/issues/6621 for further details.
  63  * This feature can be activated by by supplying one of the following jvm command line arguments:
  64  * <ul>
  65  * <li><code>-DregistrationCreate=iapt</code>: create all iapt Registrations if missing</li>
  66  * <li><code>-DregistrationWipeout=iapt</code>: remove all iapt Registrations</li>
  67  * <li><code>-DregistrationWipeout=all</code>: remove all Registrations</li>
  68  * </ul>
  69  * The <code>-DregistrationWipeout</code> commands are executed before the <code>-DregistrationCreate</code> and will not change the name and type designations.
  70  *
  71  * @author a.kohlbecker
  72  * @since May 9, 2017
  73  *
  74  */
  75 public class RegistrationRequiredDataInserter extends AbstractDataInserter {
  76
  77     protected static final String PARAM_NAME_CREATE = "registrationCreate";
  78
  79     protected static final String PARAM_NAME_WIPEOUT = "registrationWipeout";
  80
  81     protected static final UUID GROUP_SUBMITTER_UUID = UUID.fromString("c468c6a7-b96c-4206-849d-5a825f806d3e");
  82
  83     protected static final UUID GROUP_CURATOR_UUID = UUID.fromString("135210d3-3db7-4a81-ab36-240444637d45");
  84
  85     private static final EnumSet<CRUD> CREATE_READ = EnumSet.of(CRUD.CREATE, CRUD.READ);
  86     private static final EnumSet<CRUD> CREATE_READ_UPDATE_DELETE = EnumSet.of(CRUD.CREATE, CRUD.READ, CRUD.UPDATE, CRUD.DELETE);
  87
  88     private static final Logger logger = Logger.getLogger(RegistrationRequiredDataInserter.class);
  89
  90     private ExtensionType extensionTypeIAPTRegData;
  91
  92     Map<String, Institution> instituteMap = new HashMap<>();
  93
  94     public static boolean commandsExecuted = false;
  95
  96     private CdmRepository repo;
  97
  98     private boolean hasRun = false;
  99
 100     public void setCdmRepository(CdmRepository repo){
 101       this.repo = repo;
 102     }
 103
 104
 105  // ==================== Registration creation ======================= //
 106
 107     /**
 108      * {@inheritDoc}
 109      */
 110     @Override
 111     public void onApplicationEvent(ContextRefreshedEvent event) {
 112
 113         if(hasRun){
 114             return;
 115         }
 116
 117         runAsAuthentication(Role.ROLE_ADMIN);
 118
 119         insertRequiredData();
 120         executeSuppliedCommands();
 121
 122         restoreAuthentication();
 123
 124         hasRun = true;
 125     }
 126
 127     /**
 128      *
 129      */
 130     @Transactional
 131     private void insertRequiredData() {
 132
 133         TransactionStatus txStatus = repo.startTransaction(false);
 134
 135         Role roleCuration = RolesAndPermissions.ROLE_CURATION;
 136         if(repo.getGrantedAuthorityService().find(roleCuration.getUuid()) == null){
 137             repo.getGrantedAuthorityService().saveOrUpdate(roleCuration.asNewGrantedAuthority());
 138         }
 139
 140         Group groupCurator = repo.getGroupService().load(GROUP_CURATOR_UUID, Arrays.asList("grantedAuthorities"));
 141         if(groupCurator == null){
 142             groupCurator = Group.NewInstance();
 143             groupCurator.setUuid(GROUP_CURATOR_UUID);
 144             groupCurator.setName("Curator");
 145         }
 146         assureGroupHas(groupCurator, new CdmAuthority(CdmPermissionClass.REGISTRATION, CREATE_READ_UPDATE_DELETE).toString());
 147         repo.getGroupService().saveOrUpdate(groupCurator);
 148
 149         Group groupSubmitter = repo.getGroupService().load(GROUP_SUBMITTER_UUID, Arrays.asList("grantedAuthorities"));
 150         if(groupSubmitter == null){
 151             groupSubmitter = Group.NewInstance();
 152             groupSubmitter.setUuid(GROUP_SUBMITTER_UUID);
 153             groupSubmitter.setName("Submitter");
 154         }
 155         assureGroupHas(groupSubmitter, new CdmAuthority(CdmPermissionClass.TAXONNAME, CREATE_READ).toString());
 156         assureGroupHas(groupSubmitter, new CdmAuthority(CdmPermissionClass.TEAMORPERSONBASE, CREATE_READ).toString());
 157         assureGroupHas(groupSubmitter, new CdmAuthority(CdmPermissionClass.REGISTRATION, CREATE_READ).toString());
 158         assureGroupHas(groupSubmitter, new CdmAuthority(CdmPermissionClass.REFERENCE, CREATE_READ).toString());
 159         assureGroupHas(groupSubmitter, new CdmAuthority(CdmPermissionClass.SPECIMENOROBSERVATIONBASE, CREATE_READ).toString());
 160         repo.getGroupService().saveOrUpdate(groupSubmitter);
 161
 162         if(repo.getTermService().find(KindOfUnitTerms.SPECIMEN().getUuid()) == null){
 163             repo.getTermService().save(KindOfUnitTerms.SPECIMEN());
 164         }
 165         if(repo.getTermService().find(KindOfUnitTerms.PUBLISHED_IMAGE().getUuid()) == null){
 166             repo.getTermService().save(KindOfUnitTerms.PUBLISHED_IMAGE());
 167         }
 168         if(repo.getTermService().find(KindOfUnitTerms.UNPUBLISHED_IMAGE().getUuid()) == null){
 169             repo.getTermService().save(KindOfUnitTerms.UNPUBLISHED_IMAGE());
 170         }
 171         if(repo.getTermService().find(KindOfUnitTerms.CULTURE_METABOLIC_INACTIVE().getUuid()) == null){
 172             repo.getTermService().save(KindOfUnitTerms.CULTURE_METABOLIC_INACTIVE());
 173         }
 174
 175         // --- remove after release 4.12.0 ------------------------------------------------------
 176         // delete old DerivationEventTypes terms which are no longer used, see #7059
 177         // UUID_PUBLISHED_IMAGE = UUID.fromString("b8cba359-4202-4741-8ed8-4f17ae94b3e3");
 178         // UUID UUID_UNPUBLISHED_IMAGE = UUID.fromString("6cd5681f-0918-4ed6-89a8-bda1480dc890");
 179         // UUID UUID_CULTURE_METABOLIC_INACTIVE = UUID.fromString("eaf1c853-ba8d-4c40-aa0a-56beac96b0d2");
 180         for(UUID uuid : new UUID[]{
 181                 UUID.fromString("b8cba359-4202-4741-8ed8-4f17ae94b3e3"),
 182                 UUID.fromString("6cd5681f-0918-4ed6-89a8-bda1480dc890"),
 183                 UUID.fromString("eaf1c853-ba8d-4c40-aa0a-56beac96b0d2")}){
 184             if(repo.getTermService().find(uuid) != null){
 185                 repo.getTermService().delete(uuid);
 186             }
 187         }
 188         // --------------------------------------------------------------------------------------
 189         txStatus.flush();
 190         repo.commitTransaction(txStatus);
 191
 192     }
 193
 194     private void assureGroupHas(Group group, String authorityString){
 195         boolean authorityExists = false;
 196
 197         for(GrantedAuthority ga : group.getGrantedAuthorities()){
 198             if((authorityExists = ga.getAuthority().equals(authorityString)) == true){
 199                 break;
 200             }
 201         }
 202         if(!authorityExists){
 203             group.addGrantedAuthority(findGrantedAuthority(authorityString));
 204         }
 205     }
 206
 207     private GrantedAuthorityImpl findGrantedAuthority(String authorityString){
 208         GrantedAuthorityImpl ga = null;
 209         try{
 210             ga = repo.getGrantedAuthorityService().findAuthorityString(authorityString);
 211         } catch (AuthenticationCredentialsNotFoundException e){
 212             e.printStackTrace();
 213         }
 214         if(ga == null){
 215             ga = GrantedAuthorityImpl.NewInstance(authorityString);
 216             repo.getGrantedAuthorityService().save(ga);
 217         }
 218         return ga;
 219     }
 220
 221     /**
 222      *
 223      */
 224
 225     private void executeSuppliedCommands() {
 226
 227         if(commandsExecuted){
 228             // do not run twice
 229             // a second run could take place during initialization of the web context
 230             return;
 231         }
 232         commandsExecuted  = true;
 233
 234         String wipeoutCmd = System.getProperty(PARAM_NAME_WIPEOUT);
 235         String createCmd = System.getProperty(PARAM_NAME_CREATE);
 236
 237         // ============ DELETE
 238         if(wipeoutCmd != null && wipeoutCmd.matches("iapt|all")){
 239
 240             boolean onlyIapt = wipeoutCmd.equals("iapt");
 241             Set<UUID> deleteCandidates = new HashSet<UUID>();
 242
 243             TransactionStatus tx = repo.startTransaction(true);
 244             List<Registration> allRegs = repo.getRegistrationService().list(null, null, null, null, null);
 245             for(Registration reg : allRegs){
 246                 if(onlyIapt){
 247                     try {
 248                         @SuppressWarnings("unchecked")
 249                         Set<String> extensions = reg.getName().getExtensions(getExtensionTypeIAPTRegData());
 250                         if(reg.getUuid() != null){
 251                             deleteCandidates.add(reg.getUuid());
 252                         }
 253                     } catch(NullPointerException e){
 254                         // IGNORE
 255                     }
 256                 } else {
 257                     if(reg.getUuid() != null){
 258                         deleteCandidates.add(reg.getUuid());
 259                     }
 260                 }
 261             }
 262             repo.commitTransaction(tx);
 263             if(!deleteCandidates.isEmpty()){
 264                 try {
 265                     repo.getRegistrationService().delete(deleteCandidates);
 266                 } catch (Exception e) {
 267                     // MySQLIntegrityConstraintViolationException happens here every second run !!!
 268                     logger.error(e);
 269                 }
 270             }
 271         }
 272
 273         // ============ CREATE
 274         int pageIndex = 0;
 275         if(createCmd != null && createCmd.equals("iapt")){
 276
 277             DateTimeFormatter dateFormat1 = org.joda.time.format.DateTimeFormat.forPattern("dd.MM.yy").withPivotYear(1950);
 278             DateTimeFormatter dateFormat2 = org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd").withPivotYear(1950);
 279
 280             TransactionStatus tx = repo.startTransaction(false);
 281             while(true) {
 282                 Pager<TaxonName> pager = repo.getNameService().page(null, 1000, pageIndex, null, null);
 283                 if(pager.getRecords().isEmpty()){
 284                     break;
 285                 }
 286                 List<Registration> newRegs = new ArrayList<>(pager.getRecords().size());
 287                 for(TaxonName name : pager.getRecords()){
 288
 289
 290
 291                     Set<String> extensionValues = name.getExtensions(getExtensionTypeIAPTRegData());
 292
 293                     // there is for sure only one
 294                     if(extensionValues.isEmpty()){
 295                         continue;
 296                     }
 297
 298                     logger.debug("IAPT Registration for " + name.getTitleCache() + " ...");
 299
 300                     String iaptJson = extensionValues.iterator().next();
 301                     try {
 302
 303                         IAPTRegData iaptData = new ObjectMapper().readValue(iaptJson, IAPTRegData.class);
 304
 305                         if(iaptData.getRegId() == null){
 306                             continue;
 307                         }
 308
 309                         DateTime regDate = null;
 310                         if(iaptData.getDate() != null){
 311                             DateTimeFormatter dateFormat;
 312                             if(iaptData.getDate().matches("\\d{4}-\\d{2}-\\d{2}")){
 313                                 dateFormat = dateFormat2;
 314                             } else {
 315                                 dateFormat = dateFormat1;
 316                             }
 317                             try {
 318                                 regDate = dateFormat.parseDateTime(iaptData.getDate());
 319                                 regDate.getYear();
 320                             } catch (Exception e) {
 321                                 logger.error("Error parsing date : " + iaptData.getDate(), e);
 322                                 continue;
 323                             }
 324                         }
 325
 326                         Registration reg = Registration.NewInstance();
 327                         reg.setStatus(RegistrationStatus.PUBLISHED);
 328                         reg.setIdentifier("http://phycobank.org/" + iaptData.getRegId());
 329                         reg.setSpecificIdentifier(iaptData.getRegId().toString());
 330                         reg.setInstitution(getInstitution(iaptData.getOffice()));
 331
 332                         boolean isPhycobankID = Integer.valueOf(reg.getSpecificIdentifier()) >= 100000;
 333
 334                         Partial youngestDate = null;
 335                         Reference youngestPub = null;
 336
 337                         // find youngest publication
 338
 339                         // NOTE:
 340                         // data imported from IAPT does not have typedesignation citations and sometimes no nomref
 341
 342                         if(isPhycobankID){
 343                             youngestPub = (Reference) name.getNomenclaturalReference();
 344                             youngestDate = partial(youngestPub.getDatePublished());
 345
 346                             if(name.getTypeDesignations() != null && !name.getTypeDesignations().isEmpty()){
 347                                 for(TypeDesignationBase td : name.getTypeDesignations()){
 348                                     if(td.getCitation() == null){
 349                                         continue;
 350                                     }
 351                                     Partial pubdate = partial(td.getCitation().getDatePublished());
 352                                     if(pubdate != null){
 353                                         if(youngestDate== null || comparePartials(youngestDate, pubdate)){
 354                                             youngestDate = pubdate;
 355                                             youngestPub = td.getCitation();
 356                                         }
 357                                     }
 358                                 }
 359                             }
 360                         }
 361
 362                         if((isPhycobankID && youngestPub == name.getNomenclaturalReference()) || !isPhycobankID) {
 363                             reg.setName(name);
 364                         } else {
 365                             logger.debug("skipping name published in older referece");
 366                         }
 367                         if(name.getTypeDesignations() != null && !name.getTypeDesignations().isEmpty()){
 368                             // do not add the collection directly to avoid "Found shared references to a collection" problem
 369                             HashSet<TypeDesignationBase> typeDesignations = new HashSet<>(name.getTypeDesignations().size());
 370                             for(TypeDesignationBase td : name.getTypeDesignations()){
 371                                 if(td.getCitation() == null && isPhycobankID){
 372                                     logger.error("Missing TypeDesignation Citation in Phycobank data");
 373                                     continue;
 374                                 }
 375                                 if((isPhycobankID && youngestPub == td.getCitation()) || !isPhycobankID){
 376                                     typeDesignations.add(td);
 377                                 } else {
 378                                     logger.debug("skipping typedesignation published in older reference");
 379                                 }
 380                             }
 381                             reg.setTypeDesignations(typeDesignations);
 382                         }
 383                         reg.setRegistrationDate(regDate);
 384                         newRegs.add(reg);
 385
 386                     } catch (JsonParseException e) {
 387                         logger.error("Error parsing IAPTRegData from extension", e);
 388                     } catch (JsonMappingException e) {
 389                         logger.error("Error mapping json from extension to IAPTRegData", e);
 390                     } catch (IOException e) {
 391                         logger.error(e);
 392                     }
 393
 394                 }
 395                 repo.getRegistrationService().save(newRegs);
 396                 tx.flush();
 397                 logger.debug("Registrations saved");
 398                 pageIndex++;
 399             }
 400             repo.commitTransaction(tx);
 401         }
 402     }
 403
 404
 405     /**
 406      * @param youngestDate
 407      * @param pubdate
 408      * @return
 409      */
 410     protected boolean comparePartials(Partial youngestDate, Partial pubdate) {
 411
 412         if(youngestDate.size() == pubdate.size()) {
 413             return youngestDate.compareTo(pubdate) < 0;
 414         }
 415         youngestDate = youngestDate.without(DateTimeFieldType.dayOfMonth());
 416         pubdate = pubdate.without(DateTimeFieldType.dayOfMonth());
 417         if(youngestDate.size() == pubdate.size()) {
 418             return youngestDate.compareTo(pubdate) < 0;
 419         }
 420         youngestDate = youngestDate.without(DateTimeFieldType.monthOfYear());
 421         pubdate = pubdate.without(DateTimeFieldType.monthOfYear());
 422         return youngestDate.compareTo(pubdate) < 0;
 423
 424     }
 425
 426
 427     /**
 428      * @param datePublished
 429      * @return
 430      */
 431     private Partial partial(TimePeriod datePublished) {
 432         if(datePublished != null){
 433             if(datePublished.getEnd() != null){
 434                 return datePublished.getEnd();
 435             } else {
 436                 return datePublished.getStart();
 437             }
 438         }
 439         return null;
 440     }
 441
 442
 443     /**
 444      * @param office
 445      * @return
 446      */
 447     private Institution getInstitution(String office) {
 448         Institution institution;
 449         if(instituteMap.containsKey(office)){
 450             institution = instituteMap.get(office);
 451         } else {
 452
 453             Pager<AgentBase> pager = repo.getAgentService().findByTitle(Institution.class, office, MatchMode.EXACT, null, null, null, null, null);
 454             if(!pager.getRecords().isEmpty()){
 455                 institution =  (Institution) pager.getRecords().get(0);
 456             } else {
 457                 Institution institute = (Institution) repo.getAgentService().save(Institution.NewNamedInstance(office));
 458                 institution = institute;
 459             }
 460             instituteMap.put(office, institution);
 461         }
 462         return institution;
 463     }
 464
 465
 466     private ExtensionType getExtensionTypeIAPTRegData() {
 467         if(extensionTypeIAPTRegData == null){
 468             extensionTypeIAPTRegData = (ExtensionType) repo.getTermService().load(UUID.fromString("9be1bfe3-6ba0-4560-af15-86971ab96e09"));
 469         }
 470         return extensionTypeIAPTRegData;
 471     }
 472
 473
 474
 475 }